Skip to content

Commit f57a49b

Browse files
jepett0jepett0
authored
Merge 23edc19 into 96dfa75 
2 parents 96dfa75 + 23edc19 commit f57a49b

File tree

7 files changed

+143
-23
lines changed

7 files changed

+143
-23
lines changed

‎ydb/core/kqp/compile_service/kqp_compile_actor.cpp

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -254,11 +254,10 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
254254
AsyncCompileResult->Continue().Apply(callback);
255255
}
256256

257-
void AddMessageToReplayLog(const TString& queryPlan) {
257+
void AddMessageToReplayLog(const TString& queryPlan, const TVector<NKikimrKqp::TKqpTableMetadataProto>& collectedSchemeData) {
258258
NJson::TJsonValue replayMessage(NJson::JSON_MAP);
259259

260260
NJson::TJsonValue tablesMeta(NJson::JSON_ARRAY);
261-
auto collectedSchemeData = Gateway->GetCollectedSchemeData();
262261
for (auto proto: collectedSchemeData) {
263262
tablesMeta.AppendValue(Base64Encode(proto.SerializeAsString()));
264263
}
@@ -372,10 +371,14 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
372371
PassAway();
373372
}
374373

375-
void FillCompileResult(std::unique_ptr<NKikimrKqp::TPreparedQuery> preparingQuery, NKikimrKqp::EQueryType queryType) {
374+
void FillCompileResult(std::unique_ptr<NKikimrKqp::TPreparedQuery> preparingQuery,
375+
NKikimrKqp::EQueryType queryType,
376+
const TVector<NKikimrKqp::TKqpTableMetadataProto>& collectedViewsMetadata = {}
377+
) {
376378
auto preparedQueryHolder = std::make_shared<TPreparedQueryHolder>(
377379
preparingQuery.release(), AppData()->FunctionRegistry);
378380
preparedQueryHolder->MutableLlvmSettings().Fill(Config, queryType);
381+
preparedQueryHolder->FillViews(collectedViewsMetadata);
379382
KqpCompileResult->PreparedQuery = preparedQueryHolder;
380383
KqpCompileResult->AllowCache = CanCacheQuery(KqpCompileResult->PreparedQuery->GetPhysicalQuery());
381384

@@ -403,10 +406,6 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
403406
Counters->ReportSqlVersion(DbCounters, *kqpResult.SqlVersion);
404407
}
405408

406-
if (status == Ydb::StatusIds::SUCCESS) {
407-
AddMessageToReplayLog(kqpResult.QueryPlan);
408-
}
409-
410409
ETableReadType maxReadType = ExtractMostHeavyReadType(kqpResult.QueryPlan);
411410

412411
auto queryType = QueryId.Settings.QueryType;
@@ -415,7 +414,14 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
415414

416415
if (status == Ydb::StatusIds::SUCCESS) {
417416
YQL_ENSURE(kqpResult.PreparingQuery);
418-
FillCompileResult(std::move(kqpResult.PreparingQuery), queryType);
417+
418+
auto collectedSchemeData = Gateway->GetCollectedSchemeData();
419+
AddMessageToReplayLog(kqpResult.QueryPlan, collectedSchemeData);
420+
EraseIf(collectedSchemeData, [](const NKikimrKqp::TKqpTableMetadataProto& metadata) {
421+
return !metadata.HasKind() || static_cast<EKikimrTableKind>(metadata.GetKind()) != EKikimrTableKind::View;
422+
});
423+
424+
FillCompileResult(std::move(kqpResult.PreparingQuery), queryType, std::move(collectedSchemeData));
419425

420426
auto now = TInstant::Now();
421427
auto duration = now - StartTime;

‎ydb/core/kqp/query_data/kqp_prepared_query.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,22 @@ void TPreparedQueryHolder::FillTables(const google::protobuf::RepeatedPtrField<
267267
}
268268
}
269269
}
270+
271+
void TPreparedQueryHolder::FillViews(const TVector<NKikimrKqp::TKqpTableMetadataProto>& viewsMetadata) {
272+
for (const auto& view : viewsMetadata) {
273+
const auto& pathId = view.GetPathId();
274+
const auto schemaVersion = view.GetSchemaVersion();
275+
const auto& tableName = view.GetName();
276+
277+
NKqpProto::TKqpTableInfo tableInfo;
278+
tableInfo.MutableTableId()->SetOwnerId(pathId.GetOwnerId());
279+
tableInfo.MutableTableId()->SetTableId(pathId.GetTableId());
280+
tableInfo.SetSchemaVersion(schemaVersion);
281+
tableInfo.SetTableName(tableName);
282+
283+
QueryViews.emplace_back(std::move(tableInfo));
284+
}
285+
}
270286

271287
bool TPreparedQueryHolder::HasTempTables(TKqpTempTablesState::TConstPtr tempTablesState, bool withSessionId) const {
272288
if (!tempTablesState) {

‎ydb/core/kqp/query_data/kqp_prepared_query.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ class TPreparedQueryHolder {
137137
std::shared_ptr<const NKikimrKqp::TPreparedQuery> Proto;
138138
std::shared_ptr<TPreparedQueryAllocHolder> Alloc;
139139
TVector<TString> QueryTables;
140+
TVector<NKqpProto::TKqpTableInfo> QueryViews;
140141
std::vector<TKqpPhyTxHolder::TConstPtr> Transactions;
141142
TIntrusivePtr<TTableConstInfoMap> TableConstInfoById;
142143

@@ -180,6 +181,10 @@ class TPreparedQueryHolder {
180181
return QueryTables;
181182
}
182183

184+
const TVector<NKqpProto::TKqpTableInfo>& GetQueryViews() const {
185+
return QueryViews;
186+
}
187+
183188
const NKqpProto::TKqpPhyQuery& GetPhysicalQuery() const {
184189
return Proto->GetPhysicalQuery();
185190
}
@@ -192,6 +197,8 @@ class TPreparedQueryHolder {
192197

193198
void FillTables(const google::protobuf::RepeatedPtrField< ::NKqpProto::TKqpPhyStage>& stages);
194199

200+
void FillViews(const TVector<NKikimrKqp::TKqpTableMetadataProto>& viewsMetadata);
201+
195202
bool HasTempTables(TKqpTempTablesState::TConstPtr tempTablesState, bool withSessionId) const;
196203
};
197204

‎ydb/core/kqp/session_actor/kqp_query_state.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,6 @@ std::unique_ptr<TEvTxProxySchemeCache::TEvNavigateKeySet> TKqpQueryState::BuildN
100100
return std::make_unique<TEvTxProxySchemeCache::TEvNavigateKeySet>(navigate.Release());
101101
}
102102

103-
104103
bool TKqpQueryState::SaveAndCheckCompileResult(TEvKqp::TEvCompileResponse* ev) {
105104
CompileResult = ev->CompileResult;
106105
YQL_ENSURE(CompileResult);

‎ydb/core/kqp/session_actor/kqp_query_state.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,16 @@ class TKqpQueryState : public TNonCopyable {
205205
TableVersions.emplace(tableId, table.GetVersion());
206206
}
207207
};
208+
auto addView = [&](const NKqpProto::TKqpTableInfo& view) {
209+
NKikimr::TTableId tableId(view.GetTableId().GetOwnerId(), view.GetTableId().GetTableId());
210+
auto it = TableVersions.find(tableId);
211+
if (it != TableVersions.end()) {
212+
Y_ENSURE(it->second == view.GetSchemaVersion());
213+
} else {
214+
TableVersions.emplace(tableId, view.GetSchemaVersion());
215+
}
216+
};
217+
208218
for (const auto& stage : phyTx.GetStages()) {
209219
for (const auto& tableOp : stage.GetTableOps()) {
210220
addTable(tableOp.GetTable());
@@ -232,6 +242,9 @@ class TKqpQueryState : public TNonCopyable {
232242
addTable(table.GetId());
233243
}
234244
}
245+
for (const auto& view : PreparedQuery->GetQueryViews()) {
246+
addView(view);
247+
}
235248
}
236249

237250
bool NeedCheckTableVersions() const {

‎ydb/core/kqp/ut/view/view_ut.cpp

Lines changed: 87 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
2+
13
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
24
#include <ydb/library/yql/sql/sql.h>
35
#include <ydb/library/yql/utils/log/log.h>
@@ -8,6 +10,7 @@
810

911
using namespace NKikimr;
1012
using namespace NKikimr::NKqp;
13+
using namespace NYdb;
1114
using namespace NYdb::NTable;
1215

1316
namespace {
@@ -75,8 +78,37 @@ TDataQueryResult ExecuteDataModificationQuery(TSession& session,
7578
return result;
7679
}
7780

78-
TString GetYsonResults(TSession& session, const TString& query, const TExecDataQuerySettings& settings = {}) {
79-
return FormatResultSetYson(ExecuteDataModificationQuery(session, query, settings).GetResultSet(0));
81+
TValue GetSingleResult(const TDataQueryResult& rawResults) {
82+
auto resultSetParser = rawResults.GetResultSetParser(0);
83+
UNIT_ASSERT(resultSetParser.TryNextRow());
84+
return resultSetParser.GetValue(0);
85+
}
86+
87+
TValue GetSingleResult(TSession& session, const TString& query, const TExecDataQuerySettings& settings = {}) {
88+
return GetSingleResult(ExecuteDataModificationQuery(session, query, settings));
89+
}
90+
91+
TInstant GetTimestamp(const TValue& value) {
92+
return TValueParser(value).GetTimestamp();
93+
}
94+
95+
int GetInteger(const TValue& value) {
96+
return TValueParser(value).GetInt32();
97+
}
98+
99+
TMaybe<bool> GetFromCache(const TQueryStats& stats) {
100+
const auto& proto = NYdb::TProtoAccessor::GetProto(stats);
101+
if (!proto.Hascompilation()) {
102+
return Nothing();
103+
}
104+
return proto.Getcompilation().Getfrom_cache();
105+
}
106+
107+
void AssertFromCache(const TMaybe<TQueryStats>& stats, bool expectedValue) {
108+
UNIT_ASSERT(stats.Defined());
109+
const auto fromCache = GetFromCache(*stats);
110+
UNIT_ASSERT_C(fromCache.Defined(), stats->ToString());
111+
UNIT_ASSERT_VALUES_EQUAL_C(*fromCache, expectedValue, stats->ToString());
80112
}
81113

82114
void CompareResults(const TDataQueryResult& first, const TDataQueryResult& second) {
@@ -384,6 +416,53 @@ Y_UNIT_TEST_SUITE(TSelectFromViewTest) {
384416
ExecuteDataDefinitionQuery(session, ReadWholeFile(pathPrefix + "drop_view.sql"));
385417
}
386418
}
419+
420+
Y_UNIT_TEST(QueryCacheIsUpdated) {
421+
TKikimrRunner kikimr(TKikimrSettings().SetWithSampleTables(false));
422+
EnableViewsFeatureFlag(kikimr);
423+
auto session = kikimr.GetTableClient().CreateSession().GetValueSync().GetSession();
424+
425+
constexpr const char* viewName = "TheView";
426+
427+
const auto getCreationQuery = [&viewName](const char* innerQuery) -> TString {
428+
return std::format(R"(
429+
CREATE VIEW {} WITH (security_invoker = TRUE) AS {};
430+
)",
431+
viewName,
432+
innerQuery
433+
);
434+
};
435+
constexpr const char* firstInnerQuery = "SELECT 1";
436+
ExecuteDataDefinitionQuery(session, getCreationQuery(firstInnerQuery));
437+
438+
const TString selectFromViewQuery = std::format(R"(
439+
SELECT * FROM {};
440+
)",
441+
viewName
442+
);
443+
TExecDataQuerySettings queryExecutionSettings;
444+
queryExecutionSettings.KeepInQueryCache(true);
445+
queryExecutionSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
446+
ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
447+
// make sure the server side cache is working by calling the same query twice
448+
const auto cachedQueryRawResult = ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
449+
AssertFromCache(cachedQueryRawResult.GetStats(), true);
450+
UNIT_ASSERT_VALUES_EQUAL(GetInteger(GetSingleResult(cachedQueryRawResult)), 1);
451+
452+
// recreate the view with a different query inside
453+
ExecuteDataDefinitionQuery(session, std::format(R"(
454+
DROP VIEW {};
455+
)",
456+
viewName
457+
)
458+
);
459+
constexpr const char* secondInnerQuery = "SELECT 2";
460+
ExecuteDataDefinitionQuery(session, getCreationQuery(secondInnerQuery));
461+
462+
const auto secondCallRawResult = ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
463+
AssertFromCache(secondCallRawResult.GetStats(), false);
464+
UNIT_ASSERT_VALUES_EQUAL(GetInteger(GetSingleResult(secondCallRawResult)), 2);
465+
}
387466
}
388467

389468
Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
@@ -414,9 +493,9 @@ Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
414493
TExecDataQuerySettings queryExecutionSettings;
415494
queryExecutionSettings.KeepInQueryCache(true);
416495
const auto executeTwice = [&](const TString& query) {
417-
return TVector<TString>{
418-
GetYsonResults(session, query, queryExecutionSettings),
419-
GetYsonResults(session, query, queryExecutionSettings)
496+
return TVector<TInstant>{
497+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings)),
498+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings))
420499
};
421500
};
422501
const auto viewResults = executeTwice(selectFromViewQuery);
@@ -455,9 +534,9 @@ Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
455534
TExecDataQuerySettings queryExecutionSettings;
456535
queryExecutionSettings.KeepInQueryCache(true);
457536
const auto executeTwice = [&](const TString& query) {
458-
return TVector<TString>{
459-
GetYsonResults(session, query, queryExecutionSettings),
460-
GetYsonResults(session, query, queryExecutionSettings)
537+
return TVector<TInstant>{
538+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings)),
539+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings))
461540
};
462541
};
463542
const auto viewResults = executeTwice(selectFromViewQuery);

‎ydb/core/protos/kqp.proto

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -111,17 +111,17 @@ message TQueryRequest {
111111
}
112112

113113
message TKqpPathIdProto {
114-
optional uint32 OwnerId = 1;
115-
optional uint32 TableId = 2;
114+
optional uint64 OwnerId = 1;
115+
optional uint64 TableId = 2;
116116
}
117117

118118
message TIndexDescriptionProto {
119119
optional string Name = 1;
120120
optional uint32 Type = 2;
121121
optional uint32 State = 3;
122-
optional uint32 SchemaVersion = 4;
123-
optional uint32 LocalPathId = 5;
124-
optional uint32 PathOwnerId = 6;
122+
optional uint64 SchemaVersion = 4;
123+
optional uint64 LocalPathId = 5;
124+
optional uint64 PathOwnerId = 6;
125125
repeated string KeyColumns = 7;
126126
repeated string DataColumns = 8;
127127
};
@@ -160,7 +160,7 @@ message TKqpTableMetadataProto {
160160
optional string Name = 3;
161161
optional string SysView = 4;
162162
optional TKqpPathIdProto PathId = 5;
163-
optional uint32 SchemaVersion = 6;
163+
optional uint64 SchemaVersion = 6;
164164
optional uint32 Kind = 7;
165165
repeated TAttributeProto Attributes = 8;
166166
repeated TKqpColumnMetadataProto Columns = 9;

0 commit comments

Comments
 (0)