Initial implementation of facade qplayer tests (#4005)

Author: vitstn

ydb/library/yql/core/facade/ya.make

@@ -18,6 +18,7 @@ PEERDIR(
     ydb/library/yql/core/url_lister/interface
     ydb/library/yql/core/url_preprocessing/interface
     ydb/library/yql/core/credentials
+    ydb/library/yql/core/qplayer/storage/interface
     ydb/library/yql/sql
     ydb/library/yql/utils/log
     ydb/library/yql/core

ydb/library/yql/core/facade/yql_facade.cpp

@@ -44,6 +44,8 @@ namespace {
 
 const size_t DEFAULT_AST_BUF_SIZE = 1024;
 const size_t DEFAULT_PLAN_BUF_SIZE = 1024;
+const TString FacadeComponent = "Facade";
+const TString SourceCodeLabel = "SourceCode";
 
 class TUrlLoader : public IUrlLoader {
 public:
@@ -311,6 +313,12 @@ TProgram::~TProgram() {
     }
 }
 
+void TProgram::SetQContext(const TQContext& qContext) {
+    YQL_PROFILE_FUNC(TRACE);
+    YQL_ENSURE(SourceSyntax_ == ESourceSyntax::Unknown);
+    QContext_ = qContext;
+}
+
 void TProgram::ConfigureYsonResultFormat(NYson::EYsonFormat format) {
     ResultFormat_ = format;
     OutputFormat_ = format;
@@ -468,12 +476,24 @@ void TProgram::AddUserDataTable(const TUserDataTable& userDataTable) {
     }
 }
 
+void TProgram::HandleSourceCode(TString& sourceCode) {
+    if (QContext_.CanWrite()) {
+        QContext_.GetWriter()->Put({FacadeComponent, SourceCodeLabel}, sourceCode).GetValueSync();
+    } else if (QContext_.CanRead()) {
+        auto loaded = QContext_.GetReader()->Get({FacadeComponent, SourceCodeLabel}).GetValueSync();
+        Y_ENSURE(loaded.Defined(), "No source code");
+        sourceCode = loaded->Value;
+    }
+}
+
 bool TProgram::ParseYql() {
     YQL_PROFILE_FUNC(TRACE);
     YQL_ENSURE(SourceSyntax_ == ESourceSyntax::Unknown);
     SourceSyntax_ = ESourceSyntax::Yql;
     SyntaxVersion_ = 1;
-    return FillParseResult(ParseAst(SourceCode_));
+    auto sourceCode = SourceCode_;
+    HandleSourceCode(sourceCode);
+    return FillParseResult(ParseAst(sourceCode));
 }
 
 bool TProgram::ParseSql() {
@@ -495,7 +515,9 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings)
     SourceSyntax_ = ESourceSyntax::Sql;
     SyntaxVersion_ = settings.SyntaxVersion;
     NYql::TWarningRules warningRules;
-    return FillParseResult(SqlToYql(SourceCode_, settings, &warningRules), &warningRules);
+    auto sourceCode = SourceCode_;
+    HandleSourceCode(sourceCode);
+    return FillParseResult(SqlToYql(sourceCode, settings, &warningRules), &warningRules);
 }
 
 bool TProgram::Compile(const TString& username, bool skipLibraries) {

ydb/library/yql/core/facade/yql_facade.h

@@ -8,6 +8,7 @@
 #include <ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.h>
 #include <ydb/library/yql/core/yql_type_annotation.h>
 #include <ydb/library/yql/core/yql_user_data.h>
+#include <ydb/library/yql/core/qplayer/storage/interface/yql_qstorage.h>
 #include <ydb/library/yql/providers/config/yql_config_provider.h>
 #include <ydb/library/yql/providers/result/provider/yql_result_provider.h>
 #include <ydb/library/yql/public/issue/yql_issue.h>
@@ -106,6 +107,8 @@ class TProgram: public TThrRefBase, private TNonCopyable
 public:
     ~TProgram();
 
+    void SetQContext(const TQContext& qContext);
+
     void AddCredentials(const TVector<std::pair<TString, TCredential>>& credentials);
     void ClearCredentials();
 
@@ -386,6 +389,7 @@ class TProgram: public TThrRefBase, private TNonCopyable
 
 private:
     std::optional<bool> CheckFallbackIssues(const TIssues& issues);
+    void HandleSourceCode(TString& sourceCode);
 
     const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_;
     const TIntrusivePtr<IRandomProvider> RandomProvider_;
@@ -445,6 +449,8 @@ class TProgram: public TThrRefBase, private TNonCopyable
     const EHiddenMode HiddenMode_ = EHiddenMode::Disable;
     THiddenQueryAborter AbortHidden_ = [](){};
     TMaybe<TString> LineageStr_;
+
+    TQContext QContext_;
 };
 
 } // namspace NYql

ydb/library/yql/core/qplayer/storage/interface/yql_qstorage.h

@@ -96,6 +96,9 @@ class TQContext {
         : Writer_(writer)
     {}
 
+    TQContext(const TQContext&) = default;
+    TQContext& operator=(const TQContext&) = default;
+
     bool CanRead() const {
         return Reader_ != nullptr;
     }
@@ -113,8 +116,8 @@ class TQContext {
     }
 
 private:
-    const IQReaderPtr Reader_;
-    const IQWriterPtr Writer_;
+    IQReaderPtr Reader_;
+    IQWriterPtr Writer_;
 };
 
 }

ydb/library/yql/core/ut/ya.make

@@ -11,6 +11,7 @@ SRCS(
     yql_library_compiler_ut.cpp
     yql_opt_utils_ut.cpp
     yql_udf_index_ut.cpp
+    yql_qplayer_ut.cpp
 )
 
 PEERDIR(
@@ -19,6 +20,7 @@ PEERDIR(
     ydb/library/yql/core
     ydb/library/yql/core/facade
     ydb/library/yql/core/services
+    ydb/library/yql/core/qplayer/storage/memory
     ydb/library/yql/public/udf
     ydb/library/yql/public/udf/service/exception_policy
     ydb/library/yql/core/type_ann

ydb/library/yql/core/ut/yql_qplayer_ut.cpp

@@ -0,0 +1,100 @@
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h>
+#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file.h>
+#include <ydb/library/yql/providers/yt/gateway/file/yql_yt_file_services.h>
+#include <ydb/library/yql/core/facade/yql_facade.h>
+#include <ydb/library/yql/core/qplayer/storage/memory/yql_qstorage_memory.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/system/user.h>
+
+using namespace NYql;
+
+bool RunProgram(const TString& sourceCode, const TQContext& qContext, bool isSql) {
+    auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry());
+    auto yqlNativeServices = NFile::TYtFileServices::Make(functionRegistry.Get(), {}, {}, "");
+    auto ytGateway = CreateYtFileGateway(yqlNativeServices);
+
+    TVector<TDataProviderInitializer> dataProvidersInit;
+    dataProvidersInit.push_back(GetYtNativeDataProviderInitializer(ytGateway));
+    TProgramFactory factory(true, functionRegistry.Get(), 0ULL, dataProvidersInit, "ut");
+
+    TProgramPtr program = factory.Create("-stdin-", sourceCode);
+    program->SetQContext(qContext);
+    if (isSql) {
+        if (!program->ParseSql()) {
+            program->PrintErrorsTo(Cerr);
+            return false;
+        } 
+    } else if (!program->ParseYql()) {
+        program->PrintErrorsTo(Cerr);
+        return false;
+    }
+
+    if (!program->Compile(GetUsername())) {
+        program->PrintErrorsTo(Cerr);
+        return false;
+    }
+
+    TProgram::TStatus status = program->Optimize(GetUsername());
+    if (status == TProgram::TStatus::Error) {
+        program->PrintErrorsTo(Cerr);
+    }
+
+    return status == TProgram::TStatus::Ok;
+}
+
+void CheckProgram(const TString& sql, bool isSql = true) {
+    auto qStorage = MakeMemoryQStorage();
+    TQContext savingCtx(qStorage->MakeWriter("foo"));
+    UNIT_ASSERT(RunProgram(sql, savingCtx, isSql));
+    savingCtx.GetWriter()->Commit().GetValueSync();
+    TQContext loadingCtx(qStorage->MakeReader("foo"));
+    UNIT_ASSERT(RunProgram("", loadingCtx, isSql));
+}
+
+Y_UNIT_TEST_SUITE(QPlayerTests) {
+    Y_UNIT_TEST(SimpleYql) {
+        auto s = R"(
+(
+(let world (block '(
+  (let output (block '(
+    (let select (block '(
+      (let core (AsList (AsStruct)))
+      (let core (PersistableRepr (block '(
+        (let projectCoreType (TypeOf core))
+        (let core (OrderedSqlProject core '((SqlProjectItem projectCoreType '"" (lambda '(row) (block '(
+          (let res (Int32 '"1"))
+          (return res)
+        ))) '('('autoName))))))
+        (return core)
+      ))))
+      (return core)
+    )))
+    (return select)
+  )))
+  (let output (Unordered output))
+  (let world (block '(
+    (let result_sink (DataSink 'result))
+    (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('('auto))) '('unordered))))
+    (return (Commit! world result_sink))
+  )))
+  (return world)
+)))
+(let world (block '(
+  (let world (CommitAll! world))
+  (return world)
+)))
+(return world)
+)
+        )";
+        
+        CheckProgram(s, false);
+    }
+
+    Y_UNIT_TEST(SimpleSql) {
+        auto s = "select 1";
+        CheckProgram(s);
+    }
+}