Skip to content

Commit a24078c

Browse files
jepett0jepett0
authored
Merge 54a8e01 into 7dd3efc 
2 parents 7dd3efc + 54a8e01 commit a24078c

31 files changed

+283
-72
lines changed

‎ydb/core/kqp/gateway/behaviour/view/manager.cpp

+11-8
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include <ydb/core/base/path.h>
44
#include <ydb/core/kqp/gateway/actors/scheme.h>
55
#include <ydb/core/kqp/gateway/utils/scheme_helpers.h>
6+
#include <ydb/core/kqp/provider/yql_kikimr_provider.h>
67
#include <ydb/core/tx/tx_proxy/proxy.h>
78

89
namespace NKikimr::NKqp {
@@ -11,13 +12,14 @@ namespace {
1112

1213
using TYqlConclusionStatus = TViewManager::TYqlConclusionStatus;
1314
using TInternalModificationContext = TViewManager::TInternalModificationContext;
15+
using TExternalModificationContext = TViewManager::TExternalModificationContext;
1416

1517
TString GetByKeyOrDefault(const NYql::TCreateObjectSettings& container, const TString& key) {
1618
const auto value = container.GetFeaturesExtractor().Extract(key);
1719
return value ? *value : TString{};
1820
}
1921

20-
TYqlConclusionStatus CheckFeatureFlag(TInternalModificationContext& context) {
22+
TYqlConclusionStatus CheckFeatureFlag(const TInternalModificationContext& context) {
2123
auto* const actorSystem = context.GetExternalData().GetActorSystem();
2224
if (!actorSystem) {
2325
ythrow yexception() << "This place needs an actor system. Please contact internal support";
@@ -48,15 +50,16 @@ std::pair<TString, TString> SplitPathByObjectId(const TString& objectId) {
4850

4951
void FillCreateViewProposal(NKikimrSchemeOp::TModifyScheme& modifyScheme,
5052
const NYql::TCreateObjectSettings& settings,
51-
const TString& database) {
53+
const TExternalModificationContext& context) {
5254

53-
const auto pathPair = SplitPathByDb(settings.GetObjectId(), database);
55+
const auto pathPair = SplitPathByDb(settings.GetObjectId(), context.GetDatabase());
5456
modifyScheme.SetWorkingDir(pathPair.first);
5557
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateView);
5658

5759
auto& viewDesc = *modifyScheme.MutableCreateView();
5860
viewDesc.SetName(pathPair.second);
5961
viewDesc.SetQueryText(GetByKeyOrDefault(settings, "query_text"));
62+
NSQLTranslation::Serialize(context.GetTranslationSettings(), *viewDesc.MutableCapturedContext());
6063

6164
if (!settings.GetFeaturesExtractor().IsFinished()) {
6265
ythrow TBadArgumentException() << "Unknown property: " << settings.GetFeaturesExtractor().GetRemainedParamsString();
@@ -92,20 +95,20 @@ NThreading::TFuture<TYqlConclusionStatus> SendSchemeRequest(TEvTxUserProxy::TEvP
9295
}
9396

9497
NThreading::TFuture<TYqlConclusionStatus> CreateView(const NYql::TCreateObjectSettings& settings,
95-
TInternalModificationContext& context) {
98+
const TInternalModificationContext& context) {
9699
auto proposal = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
97100
proposal->Record.SetDatabaseName(context.GetExternalData().GetDatabase());
98101
if (context.GetExternalData().GetUserToken()) {
99102
proposal->Record.SetUserToken(context.GetExternalData().GetUserToken()->GetSerializedToken());
100103
}
101104
auto& schemeTx = *proposal->Record.MutableTransaction()->MutableModifyScheme();
102-
FillCreateViewProposal(schemeTx, settings, context.GetExternalData().GetDatabase());
105+
FillCreateViewProposal(schemeTx, settings, context.GetExternalData());
103106

104107
return SendSchemeRequest(proposal.Release(), context.GetExternalData().GetActorSystem(), true);
105108
}
106109

107110
NThreading::TFuture<TYqlConclusionStatus> DropView(const NYql::TDropObjectSettings& settings,
108-
TInternalModificationContext& context) {
111+
const TInternalModificationContext& context) {
109112
auto proposal = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
110113
proposal->Record.SetDatabaseName(context.GetExternalData().GetDatabase());
111114
if (context.GetExternalData().GetUserToken()) {
@@ -119,8 +122,8 @@ NThreading::TFuture<TYqlConclusionStatus> DropView(const NYql::TDropObjectSettin
119122

120123
void PrepareCreateView(NKqpProto::TKqpSchemeOperation& schemeOperation,
121124
const NYql::TObjectSettingsImpl& settings,
122-
TInternalModificationContext& context) {
123-
FillCreateViewProposal(*schemeOperation.MutableCreateView(), settings, context.GetExternalData().GetDatabase());
125+
const TInternalModificationContext& context) {
126+
FillCreateViewProposal(*schemeOperation.MutableCreateView(), settings, context.GetExternalData());
124127
}
125128

126129
void PrepareDropView(NKqpProto::TKqpSchemeOperation& schemeOperation,

‎ydb/core/kqp/gateway/behaviour/view/ya.make

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ SRCS(
88
PEERDIR(
99
ydb/core/base
1010
ydb/core/kqp/gateway/actors
11+
ydb/core/kqp/provider
1112
ydb/core/tx/tx_proxy
1213
ydb/services/metadata/abstract
1314
ydb/services/metadata/manager

‎ydb/core/kqp/gateway/kqp_metadata_loader.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ TTableMetadataResult GetViewMetadataResult(
302302
metadata->SchemaVersion = description.GetVersion();
303303
metadata->Kind = NYql::EKikimrTableKind::View;
304304
metadata->Attributes = schemeEntry.Attributes;
305-
metadata->ViewPersistedData = {description.GetQueryText()};
305+
metadata->ViewPersistedData = {description.GetQueryText(), description.GetCapturedContext()};
306306

307307
return builtResult;
308308
}

‎ydb/core/kqp/host/kqp_gateway_proxy.cpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -1259,6 +1259,7 @@ class TKqpGatewayProxy : public IKikimrGateway {
12591259
if (SessionCtx->GetUserToken()) {
12601260
context.SetUserToken(*SessionCtx->GetUserToken());
12611261
}
1262+
context.SetTranslationSettings(SessionCtx->Query().TranslationSettings);
12621263

12631264
auto& phyTx = phyTxRemover.Capture(SessionCtx->Query().PreparingQuery->MutablePhysicalQuery());
12641265
phyTx.SetType(NKqpProto::TKqpPhyTx::TYPE_SCHEME);
@@ -2179,7 +2180,7 @@ class TKqpGatewayProxy : public IKikimrGateway {
21792180
if (cluster != SessionCtx->GetCluster()) {
21802181
return MakeFuture(ResultFromError<TGenericResult>("Invalid cluster: " + cluster));
21812182
}
2182-
2183+
21832184
NKqpProto::TKqpAnalyzeOperation analyzeTx;
21842185
analyzeTx.SetTablePath(settings.TablePath);
21852186
for (const auto& column: settings.Columns) {
@@ -2192,7 +2193,7 @@ class TKqpGatewayProxy : public IKikimrGateway {
21922193
phyTx.SetType(NKqpProto::TKqpPhyTx::TYPE_SCHEME);
21932194

21942195
phyTx.MutableSchemeOperation()->MutableAnalyzeTable()->Swap(&analyzeTx);
2195-
2196+
21962197
TGenericResult result;
21972198
result.SetSuccess();
21982199
return MakeFuture(result);

‎ydb/core/kqp/host/kqp_host.cpp

+13-1
Original file line numberDiff line numberDiff line change
@@ -1231,7 +1231,19 @@ class TKqpHost : public IKqpHost {
12311231
.SetQueryParameters(query.ParameterTypes)
12321232
.SetApplicationName(ApplicationName)
12331233
.SetIsEnablePgSyntax(SessionCtx->Config().FeatureFlags.GetEnablePgSyntax());
1234-
auto astRes = ParseQuery(query.Text, isSql, sqlVersion, TypesCtx->DeprecatedSQL, ctx, settingsBuilder, result.KeepInCache, result.CommandTagName);
1234+
NSQLTranslation::TTranslationSettings effectiveSettings;
1235+
auto astRes = ParseQuery(
1236+
query.Text,
1237+
isSql,
1238+
sqlVersion,
1239+
TypesCtx->DeprecatedSQL,
1240+
ctx,
1241+
settingsBuilder,
1242+
result.KeepInCache,
1243+
result.CommandTagName,
1244+
&effectiveSettings
1245+
);
1246+
SessionCtx->Query().TranslationSettings = std::move(effectiveSettings);
12351247
if (astRes.ActualSyntaxType == NYql::ESyntaxType::Pg) {
12361248
SessionCtx->Config().IndexAutoChooserMode = NKikimrConfig::TTableServiceConfig_EIndexAutoChooseMode::TTableServiceConfig_EIndexAutoChooseMode_MAX_USED_PREFIX;
12371249
}

‎ydb/core/kqp/host/kqp_translate.cpp

+4-3
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type) {
5454
YQL_ENSURE(false, "Unexpected query type: " << type);
5555
}
5656
}
57-
57+
5858
NSQLTranslation::TTranslationSettings TKqpTranslationSettingsBuilder::Build(NYql::TExprContext& ctx) {
5959
NSQLTranslation::TTranslationSettings settings;
6060
settings.PgParser = UsePgParser && *UsePgParser;
@@ -154,13 +154,14 @@ NSQLTranslation::TTranslationSettings TKqpTranslationSettingsBuilder::Build(NYql
154154
}
155155

156156
NYql::TAstParseResult ParseQuery(const TString& queryText, bool isSql, TMaybe<ui16>& sqlVersion, bool& deprecatedSQL,
157-
NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName) {
157+
NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName,
158+
NSQLTranslation::TTranslationSettings* effectiveSettings) {
158159
NYql::TAstParseResult astRes;
159160
settingsBuilder.SetSqlVersion(sqlVersion);
160161
if (isSql) {
161162
auto settings = settingsBuilder.Build(ctx);
162163
NYql::TStmtParseInfo stmtParseInfo;
163-
auto ast = NSQLTranslation::SqlToYql(queryText, settings, nullptr, &stmtParseInfo);
164+
auto ast = NSQLTranslation::SqlToYql(queryText, settings, nullptr, &stmtParseInfo, effectiveSettings);
164165
deprecatedSQL = (ast.ActualSyntaxType == NYql::ESyntaxType::YQLv0);
165166
sqlVersion = ast.ActualSyntaxType == NYql::ESyntaxType::YQLv1 ? 1 : 0;
166167
keepInCache = stmtParseInfo.KeepInCache;

‎ydb/core/kqp/host/kqp_translate.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,8 @@ NSQLTranslation::EBindingsMode RemapBindingsMode(NKikimrConfig::TTableServiceCon
9191
NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type);
9292

9393
NYql::TAstParseResult ParseQuery(const TString& queryText, bool isSql, TMaybe<ui16>& sqlVersion, bool& deprecatedSQL,
94-
NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName);
94+
NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName,
95+
NSQLTranslation::TTranslationSettings* effectiveSettings = nullptr);
9596

9697
TVector<TQueryAst> ParseStatements(const TString& queryText, const TMaybe<Ydb::Query::Syntax>& syntax, bool isSql, TKqpTranslationSettingsBuilder& settingsBuilder, bool perStatementExecution);
9798

‎ydb/core/kqp/provider/rewrite_io_utils.cpp

+9-7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "rewrite_io_utils.h"
22

33
#include <ydb/core/kqp/provider/yql_kikimr_expr_nodes.h>
4+
#include <ydb/core/kqp/provider/yql_kikimr_provider.h>
45
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
56
#include <ydb/library/yql/core/yql_expr_optimize.h>
67
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
@@ -16,16 +17,17 @@ using namespace NNodes;
1617
constexpr const char* QueryGraphNodeSignature = "SavedQueryGraph";
1718

1819
TExprNode::TPtr CompileViewQuery(
19-
const TString& query,
2020
TExprContext& ctx,
2121
NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder,
22-
IModuleResolver::TPtr moduleResolver
22+
IModuleResolver::TPtr moduleResolver,
23+
const TViewPersistedData& viewData
2324
) {
2425
auto translationSettings = settingsBuilder.Build(ctx);
2526
translationSettings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW;
27+
NSQLTranslation::Deserialize(viewData.CapturedContext, translationSettings);
2628

2729
TAstParseResult queryAst;
28-
queryAst = NSQLTranslation::SqlToYql(query, translationSettings);
30+
queryAst = NSQLTranslation::SqlToYql(viewData.QueryText, translationSettings);
2931

3032
ctx.IssueManager.AddIssues(queryAst.Issues);
3133
if (!queryAst.IsOk()) {
@@ -116,9 +118,9 @@ TExprNode::TPtr FindTopLevelRead(const TExprNode::TPtr& queryGraph) {
116118
TExprNode::TPtr RewriteReadFromView(
117119
const TExprNode::TPtr& node,
118120
TExprContext& ctx,
119-
const TString& query,
120121
NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder,
121-
IModuleResolver::TPtr moduleResolver
122+
IModuleResolver::TPtr moduleResolver,
123+
const TViewPersistedData& viewData
122124
) {
123125
YQL_PROFILE_FUNC(DEBUG);
124126

@@ -127,7 +129,7 @@ TExprNode::TPtr RewriteReadFromView(
127129

128130
TExprNode::TPtr queryGraph = FindSavedQueryGraph(readNode.Ptr());
129131
if (!queryGraph) {
130-
queryGraph = CompileViewQuery(query, ctx, settingsBuilder, moduleResolver);
132+
queryGraph = CompileViewQuery(ctx, settingsBuilder, moduleResolver, viewData);
131133
if (!queryGraph) {
132134
ctx.AddError(TIssue(ctx.GetPosition(readNode.Pos()),
133135
"The query stored in the view cannot be compiled."));
@@ -151,4 +153,4 @@ TExprNode::TPtr RewriteReadFromView(
151153
return Build<TCoLeft>(ctx, node->Pos()).Input(topLevelRead).Done().Ptr();
152154
}
153155

154-
}
156+
}

‎ydb/core/kqp/provider/rewrite_io_utils.h

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22

33
#include <ydb/core/kqp/host/kqp_translate.h>
4+
#include <ydb/core/kqp/provider/yql_kikimr_gateway.h>
45
#include <ydb/library/yql/ast/yql_expr.h>
56

67
namespace NYql {
@@ -10,9 +11,9 @@ TExprNode::TPtr FindTopLevelRead(const TExprNode::TPtr& queryGraph);
1011
TExprNode::TPtr RewriteReadFromView(
1112
const TExprNode::TPtr& node,
1213
TExprContext& ctx,
13-
const TString& query,
1414
NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder,
15-
IModuleResolver::TPtr moduleResolver
15+
IModuleResolver::TPtr moduleResolver,
16+
const TViewPersistedData& viewData
1617
);
1718

18-
}
19+
}

‎ydb/core/kqp/provider/yql_kikimr_datasink.cpp

+4-4
Original file line numberDiff line numberDiff line change
@@ -186,9 +186,9 @@ class TKiSinkIntentDeterminationTransformer: public TKiSinkVisitorTransformer {
186186
}
187187

188188
TStatus HandleDropObject(TKiDropObject node, TExprContext& ctx) override {
189-
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder()
190-
<< "DropObject is not yet implemented for intent determination transformer"));
191-
return TStatus::Error;
189+
Y_UNUSED(node);
190+
Y_UNUSED(ctx);
191+
return TStatus::Ok;
192192
}
193193

194194
TStatus HandleCreateGroup(TKiCreateGroup node, TExprContext& ctx) override {
@@ -893,7 +893,7 @@ class TKikimrDataSink : public TDataProviderBase
893893

894894
if (tableDesc.Metadata->Kind == EKikimrTableKind::Datashard && mode == "analyze") {
895895
ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << static_cast<TStringBuf>(mode) << " is not supported for oltp tables."));
896-
return true;
896+
return true;
897897
}
898898

899899
return false;

‎ydb/core/kqp/provider/yql_kikimr_datasource.cpp

+4-3
Original file line numberDiff line numberDiff line change
@@ -771,16 +771,17 @@ class TKikimrDataSource : public TDataProviderBase {
771771
.Repeat(TExprStep::LoadTablesMetadata)
772772
.Repeat(TExprStep::RewriteIO);
773773

774-
const auto& query = tableDesc.Metadata->ViewPersistedData.QueryText;
774+
const auto& viewData = tableDesc.Metadata->ViewPersistedData;
775+
775776
NKqp::TKqpTranslationSettingsBuilder settingsBuilder(
776777
SessionCtx->Query().Type,
777778
SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(),
778779
cluster,
779-
query,
780+
viewData.QueryText,
780781
SessionCtx->Config().BindingsMode,
781782
GUCSettings
782783
);
783-
return RewriteReadFromView(node, ctx, query, settingsBuilder, Types.Modules);
784+
return RewriteReadFromView(node, ctx, settingsBuilder, Types.Modules, viewData);
784785
}
785786
}
786787

‎ydb/core/kqp/provider/yql_kikimr_gateway.h

+2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <ydb/core/protos/flat_scheme_op.pb.h>
2323
#include <ydb/core/protos/kqp.pb.h>
2424
#include <ydb/core/protos/kqp_stats.pb.h>
25+
#include <ydb/core/protos/yql_translation_settings.pb.h>
2526
#include <ydb/core/scheme/scheme_types_proto.h>
2627

2728
#include <library/cpp/json/json_reader.h>
@@ -446,6 +447,7 @@ enum EMetaSerializationType : ui64 {
446447

447448
struct TViewPersistedData {
448449
TString QueryText;
450+
NYql::NProto::TTranslationSettings CapturedContext;
449451
};
450452

451453
struct TKikimrTableMetadata : public TThrRefBase {

‎ydb/core/kqp/provider/yql_kikimr_provider.cpp

+34-1
Original file line numberDiff line numberDiff line change
@@ -838,7 +838,7 @@ Ydb::Table::VectorIndexSettings_Distance VectorIndexSettingsParseDistance(std::s
838838
return Ydb::Table::VectorIndexSettings::DISTANCE_MANHATTAN;
839839
else if (distance == "euclidean")
840840
return Ydb::Table::VectorIndexSettings::DISTANCE_EUCLIDEAN;
841-
else
841+
else
842842
YQL_ENSURE(false, "Wrong index setting distance: " << distance);
843843
};
844844

@@ -998,3 +998,36 @@ TCoNameValueTupleList TKiExecDataQuerySettings::BuildNode(TExprContext& ctx, TPo
998998
}
999999

10001000
} // namespace NYql
1001+
1002+
namespace NSQLTranslation {
1003+
1004+
void Serialize(const TTranslationSettings& settings, NYql::NProto::TTranslationSettings& serializedSettings) {
1005+
serializedSettings.SetPathPrefix(settings.PathPrefix);
1006+
serializedSettings.SetSyntaxVersion(settings.SyntaxVersion);
1007+
serializedSettings.SetAnsiLexer(settings.AnsiLexer);
1008+
serializedSettings.SetPgParser(settings.PgParser);
1009+
1010+
auto* pragmas = serializedSettings.MutablePragmas();
1011+
pragmas->Clear();
1012+
pragmas->Add(settings.Flags.begin(), settings.Flags.end());
1013+
}
1014+
1015+
void Deserialize(const NYql::NProto::TTranslationSettings& serializedSettings, TTranslationSettings& settings) {
1016+
#define DeserializeSetting(settingName) \
1017+
if (serializedSettings.Has##settingName()) { \
1018+
settings.settingName = serializedSettings.Get##settingName(); \
1019+
}
1020+
1021+
DeserializeSetting(PathPrefix);
1022+
DeserializeSetting(SyntaxVersion);
1023+
DeserializeSetting(AnsiLexer);
1024+
DeserializeSetting(PgParser);
1025+
1026+
#undef DeserializeSetting
1027+
1028+
// overwrite existing pragmas
1029+
settings.Flags.clear();
1030+
settings.Flags.insert(serializedSettings.GetPragmas().begin(), serializedSettings.GetPragmas().end());
1031+
}
1032+
1033+
}

‎ydb/core/kqp/provider/yql_kikimr_provider.h

+10
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,8 @@ struct TKikimrQueryContext : TThrRefBase {
125125
// we do not want add extra life time for query context here
126126
std::shared_ptr<NKikimr::NGRpcService::IRequestCtxMtSafe> RpcCtx;
127127

128+
NSQLTranslation::TTranslationSettings TranslationSettings;
129+
128130
void Reset() {
129131
PrepareOnly = false;
130132
SuppressDdlChecks = false;
@@ -143,6 +145,7 @@ struct TKikimrQueryContext : TThrRefBase {
143145

144146
RlPath.Clear();
145147
RpcCtx.reset();
148+
TranslationSettings = NSQLTranslation::TTranslationSettings();
146149
}
147150
};
148151

@@ -568,3 +571,10 @@ TIntrusivePtr<IDataProvider> CreateKikimrDataSink(
568571
TIntrusivePtr<IKikimrQueryExecutor> queryExecutor);
569572

570573
} // namespace NYql
574+
575+
namespace NSQLTranslation {
576+
577+
void Serialize(const TTranslationSettings& settings, NYql::NProto::TTranslationSettings& serializedSettings);
578+
void Deserialize(const NYql::NProto::TTranslationSettings& serializedSettings, TTranslationSettings& settings);
579+
580+
}

0 commit comments

Comments
 (0)