Skip to content

Commit fbcefe2

Browse files
jepett0jepett0
committed
Capture TablePathPrefix (and other parts of the parser context) in CREATE VIEW (ydb-platform#8991)
1 parent adc27fd commit fbcefe2

31 files changed

+274
-63
lines changed

‎ydb/core/kqp/gateway/behaviour/view/manager.cpp

+11-8
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include <ydb/core/base/path.h>
44
#include <ydb/core/kqp/gateway/actors/scheme.h>
55
#include <ydb/core/kqp/gateway/utils/scheme_helpers.h>
6+
#include <ydb/core/kqp/provider/yql_kikimr_provider.h>
67
#include <ydb/core/tx/tx_proxy/proxy.h>
78

89
namespace NKikimr::NKqp {
@@ -11,13 +12,14 @@ namespace {
1112

1213
using TYqlConclusionStatus = TViewManager::TYqlConclusionStatus;
1314
using TInternalModificationContext = TViewManager::TInternalModificationContext;
15+
using TExternalModificationContext = TViewManager::TExternalModificationContext;
1416

1517
TString GetByKeyOrDefault(const NYql::TCreateObjectSettings& container, const TString& key) {
1618
const auto value = container.GetFeaturesExtractor().Extract(key);
1719
return value ? *value : TString{};
1820
}
1921

20-
TYqlConclusionStatus CheckFeatureFlag(TInternalModificationContext& context) {
22+
TYqlConclusionStatus CheckFeatureFlag(const TInternalModificationContext& context) {
2123
auto* const actorSystem = context.GetExternalData().GetActorSystem();
2224
if (!actorSystem) {
2325
ythrow yexception() << "This place needs an actor system. Please contact internal support";
@@ -48,15 +50,16 @@ std::pair<TString, TString> SplitPathByObjectId(const TString& objectId) {
4850

4951
void FillCreateViewProposal(NKikimrSchemeOp::TModifyScheme& modifyScheme,
5052
const NYql::TCreateObjectSettings& settings,
51-
const TString& database) {
53+
const TExternalModificationContext& context) {
5254

53-
const auto pathPair = SplitPathByDb(settings.GetObjectId(), database);
55+
const auto pathPair = SplitPathByDb(settings.GetObjectId(), context.GetDatabase());
5456
modifyScheme.SetWorkingDir(pathPair.first);
5557
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateView);
5658

5759
auto& viewDesc = *modifyScheme.MutableCreateView();
5860
viewDesc.SetName(pathPair.second);
5961
viewDesc.SetQueryText(GetByKeyOrDefault(settings, "query_text"));
62+
NSQLTranslation::Serialize(context.GetTranslationSettings(), *viewDesc.MutableCapturedContext());
6063

6164
if (!settings.GetFeaturesExtractor().IsFinished()) {
6265
ythrow TBadArgumentException() << "Unknown property: " << settings.GetFeaturesExtractor().GetRemainedParamsString();
@@ -92,20 +95,20 @@ NThreading::TFuture<TYqlConclusionStatus> SendSchemeRequest(TEvTxUserProxy::TEvP
9295
}
9396

9497
NThreading::TFuture<TYqlConclusionStatus> CreateView(const NYql::TCreateObjectSettings& settings,
95-
TInternalModificationContext& context) {
98+
const TInternalModificationContext& context) {
9699
auto proposal = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
97100
proposal->Record.SetDatabaseName(context.GetExternalData().GetDatabase());
98101
if (context.GetExternalData().GetUserToken()) {
99102
proposal->Record.SetUserToken(context.GetExternalData().GetUserToken()->GetSerializedToken());
100103
}
101104
auto& schemeTx = *proposal->Record.MutableTransaction()->MutableModifyScheme();
102-
FillCreateViewProposal(schemeTx, settings, context.GetExternalData().GetDatabase());
105+
FillCreateViewProposal(schemeTx, settings, context.GetExternalData());
103106

104107
return SendSchemeRequest(proposal.Release(), context.GetExternalData().GetActorSystem(), true);
105108
}
106109

107110
NThreading::TFuture<TYqlConclusionStatus> DropView(const NYql::TDropObjectSettings& settings,
108-
TInternalModificationContext& context) {
111+
const TInternalModificationContext& context) {
109112
auto proposal = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
110113
proposal->Record.SetDatabaseName(context.GetExternalData().GetDatabase());
111114
if (context.GetExternalData().GetUserToken()) {
@@ -119,8 +122,8 @@ NThreading::TFuture<TYqlConclusionStatus> DropView(const NYql::TDropObjectSettin
119122

120123
void PrepareCreateView(NKqpProto::TKqpSchemeOperation& schemeOperation,
121124
const NYql::TObjectSettingsImpl& settings,
122-
TInternalModificationContext& context) {
123-
FillCreateViewProposal(*schemeOperation.MutableCreateView(), settings, context.GetExternalData().GetDatabase());
125+
const TInternalModificationContext& context) {
126+
FillCreateViewProposal(*schemeOperation.MutableCreateView(), settings, context.GetExternalData());
124127
}
125128

126129
void PrepareDropView(NKqpProto::TKqpSchemeOperation& schemeOperation,

‎ydb/core/kqp/gateway/behaviour/view/ya.make

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ SRCS(
88
PEERDIR(
99
ydb/core/base
1010
ydb/core/kqp/gateway/actors
11+
ydb/core/kqp/provider
1112
ydb/core/tx/tx_proxy
1213
ydb/services/metadata/abstract
1314
ydb/services/metadata/manager

‎ydb/core/kqp/gateway/kqp_metadata_loader.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ TTableMetadataResult GetViewMetadataResult(
303303
metadata->SchemaVersion = description.GetVersion();
304304
metadata->Kind = NYql::EKikimrTableKind::View;
305305
metadata->Attributes = schemeEntry.Attributes;
306-
metadata->ViewPersistedData = {description.GetQueryText()};
306+
metadata->ViewPersistedData = {description.GetQueryText(), description.GetCapturedContext()};
307307

308308
return builtResult;
309309
}

‎ydb/core/kqp/host/kqp_gateway_proxy.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -1266,6 +1266,7 @@ class TKqpGatewayProxy : public IKikimrGateway {
12661266
if (SessionCtx->GetUserToken()) {
12671267
context.SetUserToken(*SessionCtx->GetUserToken());
12681268
}
1269+
context.SetTranslationSettings(SessionCtx->Query().TranslationSettings);
12691270

12701271
auto& phyTx = phyTxRemover.Capture(SessionCtx->Query().PreparingQuery->MutablePhysicalQuery());
12711272
phyTx.SetType(NKqpProto::TKqpPhyTx::TYPE_SCHEME);

‎ydb/core/kqp/host/kqp_host.cpp

+13-1
Original file line numberDiff line numberDiff line change
@@ -1228,7 +1228,19 @@ class TKqpHost : public IKqpHost {
12281228
.SetQueryParameters(query.ParameterTypes)
12291229
.SetApplicationName(ApplicationName)
12301230
.SetIsEnablePgSyntax(SessionCtx->Config().FeatureFlags.GetEnablePgSyntax());
1231-
auto astRes = ParseQuery(query.Text, isSql, sqlVersion, TypesCtx->DeprecatedSQL, ctx, settingsBuilder, result.KeepInCache, result.CommandTagName);
1231+
NSQLTranslation::TTranslationSettings effectiveSettings;
1232+
auto astRes = ParseQuery(
1233+
query.Text,
1234+
isSql,
1235+
sqlVersion,
1236+
TypesCtx->DeprecatedSQL,
1237+
ctx,
1238+
settingsBuilder,
1239+
result.KeepInCache,
1240+
result.CommandTagName,
1241+
&effectiveSettings
1242+
);
1243+
SessionCtx->Query().TranslationSettings = std::move(effectiveSettings);
12321244
if (astRes.ActualSyntaxType == NYql::ESyntaxType::Pg) {
12331245
SessionCtx->Config().IndexAutoChooserMode = NKikimrConfig::TTableServiceConfig_EIndexAutoChooseMode::TTableServiceConfig_EIndexAutoChooseMode_MAX_USED_PREFIX;
12341246
}

‎ydb/core/kqp/host/kqp_translate.cpp

+4-3
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type) {
5454
YQL_ENSURE(false, "Unexpected query type: " << type);
5555
}
5656
}
57-
57+
5858
NSQLTranslation::TTranslationSettings TKqpTranslationSettingsBuilder::Build(NYql::TExprContext& ctx) {
5959
NSQLTranslation::TTranslationSettings settings;
6060
settings.PgParser = UsePgParser && *UsePgParser;
@@ -154,13 +154,14 @@ NSQLTranslation::TTranslationSettings TKqpTranslationSettingsBuilder::Build(NYql
154154
}
155155

156156
NYql::TAstParseResult ParseQuery(const TString& queryText, bool isSql, TMaybe<ui16>& sqlVersion, bool& deprecatedSQL,
157-
NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName) {
157+
NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName,
158+
NSQLTranslation::TTranslationSettings* effectiveSettings) {
158159
NYql::TAstParseResult astRes;
159160
settingsBuilder.SetSqlVersion(sqlVersion);
160161
if (isSql) {
161162
auto settings = settingsBuilder.Build(ctx);
162163
NYql::TStmtParseInfo stmtParseInfo;
163-
auto ast = NSQLTranslation::SqlToYql(queryText, settings, nullptr, &stmtParseInfo);
164+
auto ast = NSQLTranslation::SqlToYql(queryText, settings, nullptr, &stmtParseInfo, effectiveSettings);
164165
deprecatedSQL = (ast.ActualSyntaxType == NYql::ESyntaxType::YQLv0);
165166
sqlVersion = ast.ActualSyntaxType == NYql::ESyntaxType::YQLv1 ? 1 : 0;
166167
keepInCache = stmtParseInfo.KeepInCache;

‎ydb/core/kqp/host/kqp_translate.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,8 @@ NSQLTranslation::EBindingsMode RemapBindingsMode(NKikimrConfig::TTableServiceCon
9191
NYql::EKikimrQueryType ConvertType(NKikimrKqp::EQueryType type);
9292

9393
NYql::TAstParseResult ParseQuery(const TString& queryText, bool isSql, TMaybe<ui16>& sqlVersion, bool& deprecatedSQL,
94-
NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName);
94+
NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder, bool& keepInCache, TMaybe<TString>& commandTagName,
95+
NSQLTranslation::TTranslationSettings* effectiveSettings = nullptr);
9596

9697
TVector<TQueryAst> ParseStatements(const TString& queryText, const TMaybe<Ydb::Query::Syntax>& syntax, bool isSql, TKqpTranslationSettingsBuilder& settingsBuilder, bool perStatementExecution);
9798

‎ydb/core/kqp/provider/rewrite_io_utils.cpp

+9-7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "rewrite_io_utils.h"
22

33
#include <ydb/core/kqp/provider/yql_kikimr_expr_nodes.h>
4+
#include <ydb/core/kqp/provider/yql_kikimr_provider.h>
45
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
56
#include <ydb/library/yql/core/yql_expr_optimize.h>
67
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
@@ -16,16 +17,17 @@ using namespace NNodes;
1617
constexpr const char* QueryGraphNodeSignature = "SavedQueryGraph";
1718

1819
TExprNode::TPtr CompileViewQuery(
19-
const TString& query,
2020
TExprContext& ctx,
2121
NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder,
22-
IModuleResolver::TPtr moduleResolver
22+
IModuleResolver::TPtr moduleResolver,
23+
const TViewPersistedData& viewData
2324
) {
2425
auto translationSettings = settingsBuilder.Build(ctx);
2526
translationSettings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW;
27+
NSQLTranslation::Deserialize(viewData.CapturedContext, translationSettings);
2628

2729
TAstParseResult queryAst;
28-
queryAst = NSQLTranslation::SqlToYql(query, translationSettings);
30+
queryAst = NSQLTranslation::SqlToYql(viewData.QueryText, translationSettings);
2931

3032
ctx.IssueManager.AddIssues(queryAst.Issues);
3133
if (!queryAst.IsOk()) {
@@ -116,9 +118,9 @@ TExprNode::TPtr FindTopLevelRead(const TExprNode::TPtr& queryGraph) {
116118
TExprNode::TPtr RewriteReadFromView(
117119
const TExprNode::TPtr& node,
118120
TExprContext& ctx,
119-
const TString& query,
120121
NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder,
121-
IModuleResolver::TPtr moduleResolver
122+
IModuleResolver::TPtr moduleResolver,
123+
const TViewPersistedData& viewData
122124
) {
123125
YQL_PROFILE_FUNC(DEBUG);
124126

@@ -127,7 +129,7 @@ TExprNode::TPtr RewriteReadFromView(
127129

128130
TExprNode::TPtr queryGraph = FindSavedQueryGraph(readNode.Ptr());
129131
if (!queryGraph) {
130-
queryGraph = CompileViewQuery(query, ctx, settingsBuilder, moduleResolver);
132+
queryGraph = CompileViewQuery(ctx, settingsBuilder, moduleResolver, viewData);
131133
if (!queryGraph) {
132134
ctx.AddError(TIssue(ctx.GetPosition(readNode.Pos()),
133135
"The query stored in the view cannot be compiled."));
@@ -151,4 +153,4 @@ TExprNode::TPtr RewriteReadFromView(
151153
return Build<TCoLeft>(ctx, node->Pos()).Input(topLevelRead).Done().Ptr();
152154
}
153155

154-
}
156+
}

‎ydb/core/kqp/provider/rewrite_io_utils.h

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22

33
#include <ydb/core/kqp/host/kqp_translate.h>
4+
#include <ydb/core/kqp/provider/yql_kikimr_gateway.h>
45
#include <ydb/library/yql/ast/yql_expr.h>
56

67
namespace NYql {
@@ -10,9 +11,9 @@ TExprNode::TPtr FindTopLevelRead(const TExprNode::TPtr& queryGraph);
1011
TExprNode::TPtr RewriteReadFromView(
1112
const TExprNode::TPtr& node,
1213
TExprContext& ctx,
13-
const TString& query,
1414
NKikimr::NKqp::TKqpTranslationSettingsBuilder& settingsBuilder,
15-
IModuleResolver::TPtr moduleResolver
15+
IModuleResolver::TPtr moduleResolver,
16+
const TViewPersistedData& viewData
1617
);
1718

18-
}
19+
}

‎ydb/core/kqp/provider/yql_kikimr_datasink.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -176,9 +176,9 @@ class TKiSinkIntentDeterminationTransformer: public TKiSinkVisitorTransformer {
176176
}
177177

178178
TStatus HandleDropObject(TKiDropObject node, TExprContext& ctx) override {
179-
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder()
180-
<< "DropObject is not yet implemented for intent determination transformer"));
181-
return TStatus::Error;
179+
Y_UNUSED(node);
180+
Y_UNUSED(ctx);
181+
return TStatus::Ok;
182182
}
183183

184184
TStatus HandleCreateGroup(TKiCreateGroup node, TExprContext& ctx) override {

‎ydb/core/kqp/provider/yql_kikimr_datasource.cpp

+4-3
Original file line numberDiff line numberDiff line change
@@ -771,16 +771,17 @@ class TKikimrDataSource : public TDataProviderBase {
771771
.Repeat(TExprStep::LoadTablesMetadata)
772772
.Repeat(TExprStep::RewriteIO);
773773

774-
const auto& query = tableDesc.Metadata->ViewPersistedData.QueryText;
774+
const auto& viewData = tableDesc.Metadata->ViewPersistedData;
775+
775776
NKqp::TKqpTranslationSettingsBuilder settingsBuilder(
776777
SessionCtx->Query().Type,
777778
SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(),
778779
cluster,
779-
query,
780+
viewData.QueryText,
780781
SessionCtx->Config().BindingsMode,
781782
GUCSettings
782783
);
783-
return RewriteReadFromView(node, ctx, query, settingsBuilder, Types.Modules);
784+
return RewriteReadFromView(node, ctx, settingsBuilder, Types.Modules, viewData);
784785
}
785786
}
786787

‎ydb/core/kqp/provider/yql_kikimr_gateway.h

+2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <ydb/core/protos/flat_scheme_op.pb.h>
2323
#include <ydb/core/protos/kqp.pb.h>
2424
#include <ydb/core/protos/kqp_stats.pb.h>
25+
#include <ydb/core/protos/yql_translation_settings.pb.h>
2526
#include <ydb/core/scheme/scheme_types_proto.h>
2627

2728
#include <library/cpp/json/json_reader.h>
@@ -406,6 +407,7 @@ enum EMetaSerializationType : ui64 {
406407

407408
struct TViewPersistedData {
408409
TString QueryText;
410+
NYql::NProto::TTranslationSettings CapturedContext;
409411
};
410412

411413
struct TKikimrTableMetadata : public TThrRefBase {

‎ydb/core/kqp/provider/yql_kikimr_provider.cpp

+33
Original file line numberDiff line numberDiff line change
@@ -964,3 +964,36 @@ TCoNameValueTupleList TKiExecDataQuerySettings::BuildNode(TExprContext& ctx, TPo
964964
}
965965

966966
} // namespace NYql
967+
968+
namespace NSQLTranslation {
969+
970+
void Serialize(const TTranslationSettings& settings, NYql::NProto::TTranslationSettings& serializedSettings) {
971+
serializedSettings.SetPathPrefix(settings.PathPrefix);
972+
serializedSettings.SetSyntaxVersion(settings.SyntaxVersion);
973+
serializedSettings.SetAnsiLexer(settings.AnsiLexer);
974+
serializedSettings.SetPgParser(settings.PgParser);
975+
976+
auto* pragmas = serializedSettings.MutablePragmas();
977+
pragmas->Clear();
978+
pragmas->Add(settings.Flags.begin(), settings.Flags.end());
979+
}
980+
981+
void Deserialize(const NYql::NProto::TTranslationSettings& serializedSettings, TTranslationSettings& settings) {
982+
#define DeserializeSetting(settingName) \
983+
if (serializedSettings.Has##settingName()) { \
984+
settings.settingName = serializedSettings.Get##settingName(); \
985+
}
986+
987+
DeserializeSetting(PathPrefix);
988+
DeserializeSetting(SyntaxVersion);
989+
DeserializeSetting(AnsiLexer);
990+
DeserializeSetting(PgParser);
991+
992+
#undef DeserializeSetting
993+
994+
// overwrite existing pragmas
995+
settings.Flags.clear();
996+
settings.Flags.insert(serializedSettings.GetPragmas().begin(), serializedSettings.GetPragmas().end());
997+
}
998+
999+
}

‎ydb/core/kqp/provider/yql_kikimr_provider.h

+10
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,8 @@ struct TKikimrQueryContext : TThrRefBase {
124124
// we do not want add extra life time for query context here
125125
std::shared_ptr<NKikimr::NGRpcService::IRequestCtxMtSafe> RpcCtx;
126126

127+
NSQLTranslation::TTranslationSettings TranslationSettings;
128+
127129
void Reset() {
128130
PrepareOnly = false;
129131
SuppressDdlChecks = false;
@@ -142,6 +144,7 @@ struct TKikimrQueryContext : TThrRefBase {
142144

143145
RlPath.Clear();
144146
RpcCtx.reset();
147+
TranslationSettings = NSQLTranslation::TTranslationSettings();
145148
}
146149
};
147150

@@ -578,3 +581,10 @@ TIntrusivePtr<IDataProvider> CreateKikimrDataSink(
578581
TIntrusivePtr<IKikimrQueryExecutor> queryExecutor);
579582

580583
} // namespace NYql
584+
585+
namespace NSQLTranslation {
586+
587+
void Serialize(const TTranslationSettings& settings, NYql::NProto::TTranslationSettings& serializedSettings);
588+
void Deserialize(const NYql::NProto::TTranslationSettings& serializedSettings, TTranslationSettings& settings);
589+
590+
}

0 commit comments

Comments
 (0)