init

Author: vitstn

ydb/library/yql/providers/config/yql_config_provider.cpp

@@ -260,14 +260,22 @@ namespace {
                         for (size_t i = 3; i < node->ChildrenSize(); ++i) {
                             if (node->Child(i)->IsCallable("EvaluateAtom")) {
                                 hasPendingEvaluations = true;
-                                return res;
+                                break;
                             }
                             if (!EnsureAtom(*node->Child(i), ctx)) {
                                 return {};
                             }
                             args.push_back(node->Child(i)->Content());
                         }
 
+                        if (hasPendingEvaluations) {
+                            if (!ValidateEvaluation(command, *node, ctx)) {
+                                return {};
+                            }
+
+                            return res;
+                        }
+
                         if (!ApplyFlag(ctx.GetPosition(node->Child(2)->Pos()), command, args, ctx)) {
                             return {};
                         }
@@ -460,6 +468,28 @@ namespace {
             return true;
         }
 
+        bool ValidateEvaluation(const TStringBuf name, const TExprNode& node, TExprContext& ctx) {
+            if (name == "AddFileByUrl" || name == "AddFolderByUrl") {
+                if (node.ChildrenSize() < 4) {
+                    ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected at least 5 arguments, but got " << node.ChildrenSize()));
+                    return false;
+                }
+
+                if (node.Child(3)->IsCallable("EvaluateAtom")) {
+                    return true;
+                }
+
+                if (!PendingEvaluationFiles.insert(TString(node.Child(3)->Content())).second) {
+                    ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Detected evaluation cycle for file: " << node.Child(3)->Content()));
+                    return false;
+                }
+
+                return true;
+            } else {
+                return true;
+            }
+        }
+
         bool ApplyFlag(const TPosition& pos, const TStringBuf name, const TVector<TStringBuf>& args, TExprContext& ctx) {
             if (!IsSettingAllowed(pos, name, ctx)) {
                 return false;
@@ -991,6 +1021,7 @@ namespace {
                 return false;
             }
 
+            PendingEvaluationFiles.erase(TString(args[0]));
             TStringBuf token = args.size() == 3 ? args[2] : TStringBuf();
             if (token) {
                 if (auto cred = Types.Credentials->FindCredential(token)) {
@@ -1110,6 +1141,7 @@ namespace {
                 return false;
             }
 
+            PendingEvaluationFiles.erase(TString(args[0]));
             TStringBuf token = args.size() == 3 ? args[2] : TStringBuf();
             if (token) {
                 if (auto cred = Types.Credentials->FindCredential(token)) {
@@ -1206,6 +1238,7 @@ namespace {
         TString Username;
         const TAllowSettingPolicy Policy;
         TOperationStatistics Statistics;
+        THashSet<TString> PendingEvaluationFiles;
     };
 }
 

ydb/library/yql/tests/sql/dq_file/part8/canondata/result.json

@@ -21,6 +21,11 @@
         }
     ],
     "test.test[action-evaluate_pure--Results]": [],
+    "test.test[action-file_cycle--Results]": [
+        {
+            "uri": "file://test.test_action-file_cycle--Results_/extracted"
+        }
+    ],
     "test.test[action-nested_eval-default.txt-Analyze]": [
         {
             "checksum": "b4dd508a329723c74293d80f0278c705",

ydb/library/yql/tests/sql/dq_file/part8/canondata/test.test_action-file_cycle--Results_/extracted

@@ -0,0 +1,5 @@
+<tmp_path>/program.sql:<main>: Error: Pre type annotation
+
+    <tmp_path>/program.sql:<main>:2:18: Error: Detected evaluation cycle for file: f
+    	pragma file("f", $p);
+	                 ^

ydb/library/yql/tests/sql/sql2yql/canondata/result.json

@@ -412,6 +412,13 @@
             "uri": "https://{canondata_backend}/1936947/659b615f15086142a8960946dabd06b519d43335/resource.tar.gz#test_sql2yql.test_action-export_action_/sql.yql"
         }
     ],
+    "test_sql2yql.test[action-file_cycle]": [
+        {
+            "checksum": "095c2c315ba26037554c056a02f5af05",
+            "size": 343,
+            "uri": "https://{canondata_backend}/1871002/4d084a25cea0c62bd09f3c9e9e7c2b56d112c5b1/resource.tar.gz#test_sql2yql.test_action-file_cycle_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[action-inline_action]": [
         {
             "checksum": "d019514b3fd1f8a7c0f9d37db4231b93",
@@ -17891,6 +17898,13 @@
             "uri": "https://{canondata_backend}/1871102/17992aa919577eec0f31ef167084ab70f41ccc80/resource.tar.gz#test_sql_format.test_action-export_action_/formatted.sql"
         }
     ],
+    "test_sql_format.test[action-file_cycle]": [
+        {
+            "checksum": "5476df82092007027246993752e24f68",
+            "size": 45,
+            "uri": "https://{canondata_backend}/1871002/4d084a25cea0c62bd09f3c9e9e7c2b56d112c5b1/resource.tar.gz#test_sql_format.test_action-file_cycle_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[action-inline_action]": [
         {
             "checksum": "7da554999ea3520183678c730ae2fffb",

ydb/library/yql/tests/sql/suites/action/file_cycle.cfg

@@ -0,0 +1,2 @@
+xfail
+

ydb/library/yql/tests/sql/suites/action/file_cycle.sql

@@ -0,0 +1,3 @@
+$p=FileContent("f");
+pragma file("f", $p);
+

ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json

@@ -20,6 +20,13 @@
             "uri": "https://{canondata_backend}/1900335/4871e48d29e4933514d3ed0c9c5d19de571eda1f/resource.tar.gz#test.test_action-evaluate_pure--Results_/results.txt"
         }
     ],
+    "test.test[action-file_cycle--Debug]": [],
+    "test.test[action-file_cycle--Plan]": [],
+    "test.test[action-file_cycle--Results]": [
+        {
+            "uri": "file://test.test_action-file_cycle--Results_/extracted"
+        }
+    ],
     "test.test[action-nested_eval-default.txt-Debug]": [
         {
             "checksum": "79af7d3cdd20c4123718aae10c3202d7",

ydb/library/yql/tests/sql/yt_native_file/part8/canondata/test.test_action-file_cycle--Results_/extracted

@@ -0,0 +1,5 @@
+<tmp_path>/program.sql:<main>: Error: Pre type annotation
+
+    <tmp_path>/program.sql:<main>:2:18: Error: Detected evaluation cycle for file: f
+    	pragma file("f", $p);
+	                 ^