Skip to content

Commit 23edc19

Browse files
jepett0jepett0
committed
Update query cache using SchemeCache if the schema version of the views
used in this query was updated Store the list of the used views alongside the query compilation results in the cache and use it to check if a recompilation is needed.
1 parent 228b90a commit 23edc19

File tree

6 files changed

+137
-17
lines changed

6 files changed

+137
-17
lines changed

‎ydb/core/kqp/compile_service/kqp_compile_actor.cpp

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -254,11 +254,10 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
254254
AsyncCompileResult->Continue().Apply(callback);
255255
}
256256

257-
void AddMessageToReplayLog(const TString& queryPlan) {
257+
void AddMessageToReplayLog(const TString& queryPlan, const TVector<NKikimrKqp::TKqpTableMetadataProto>& collectedSchemeData) {
258258
NJson::TJsonValue replayMessage(NJson::JSON_MAP);
259259

260260
NJson::TJsonValue tablesMeta(NJson::JSON_ARRAY);
261-
auto collectedSchemeData = Gateway->GetCollectedSchemeData();
262261
for (auto proto: collectedSchemeData) {
263262
tablesMeta.AppendValue(Base64Encode(proto.SerializeAsString()));
264263
}
@@ -372,10 +371,14 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
372371
PassAway();
373372
}
374373

375-
void FillCompileResult(std::unique_ptr<NKikimrKqp::TPreparedQuery> preparingQuery, NKikimrKqp::EQueryType queryType) {
374+
void FillCompileResult(std::unique_ptr<NKikimrKqp::TPreparedQuery> preparingQuery,
375+
NKikimrKqp::EQueryType queryType,
376+
const TVector<NKikimrKqp::TKqpTableMetadataProto>& collectedViewsMetadata = {}
377+
) {
376378
auto preparedQueryHolder = std::make_shared<TPreparedQueryHolder>(
377379
preparingQuery.release(), AppData()->FunctionRegistry);
378380
preparedQueryHolder->MutableLlvmSettings().Fill(Config, queryType);
381+
preparedQueryHolder->FillViews(collectedViewsMetadata);
379382
KqpCompileResult->PreparedQuery = preparedQueryHolder;
380383
KqpCompileResult->AllowCache = CanCacheQuery(KqpCompileResult->PreparedQuery->GetPhysicalQuery());
381384

@@ -403,10 +406,6 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
403406
Counters->ReportSqlVersion(DbCounters, *kqpResult.SqlVersion);
404407
}
405408

406-
if (status == Ydb::StatusIds::SUCCESS) {
407-
AddMessageToReplayLog(kqpResult.QueryPlan);
408-
}
409-
410409
ETableReadType maxReadType = ExtractMostHeavyReadType(kqpResult.QueryPlan);
411410

412411
auto queryType = QueryId.Settings.QueryType;
@@ -415,7 +414,14 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
415414

416415
if (status == Ydb::StatusIds::SUCCESS) {
417416
YQL_ENSURE(kqpResult.PreparingQuery);
418-
FillCompileResult(std::move(kqpResult.PreparingQuery), queryType);
417+
418+
auto collectedSchemeData = Gateway->GetCollectedSchemeData();
419+
AddMessageToReplayLog(kqpResult.QueryPlan, collectedSchemeData);
420+
EraseIf(collectedSchemeData, [](const NKikimrKqp::TKqpTableMetadataProto& metadata) {
421+
return !metadata.HasKind() || static_cast<EKikimrTableKind>(metadata.GetKind()) != EKikimrTableKind::View;
422+
});
423+
424+
FillCompileResult(std::move(kqpResult.PreparingQuery), queryType, std::move(collectedSchemeData));
419425

420426
auto now = TInstant::Now();
421427
auto duration = now - StartTime;

‎ydb/core/kqp/query_data/kqp_prepared_query.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,22 @@ void TPreparedQueryHolder::FillTables(const google::protobuf::RepeatedPtrField<
267267
}
268268
}
269269
}
270+
271+
void TPreparedQueryHolder::FillViews(const TVector<NKikimrKqp::TKqpTableMetadataProto>& viewsMetadata) {
272+
for (const auto& view : viewsMetadata) {
273+
const auto& pathId = view.GetPathId();
274+
const auto schemaVersion = view.GetSchemaVersion();
275+
const auto& tableName = view.GetName();
276+
277+
NKqpProto::TKqpTableInfo tableInfo;
278+
tableInfo.MutableTableId()->SetOwnerId(pathId.GetOwnerId());
279+
tableInfo.MutableTableId()->SetTableId(pathId.GetTableId());
280+
tableInfo.SetSchemaVersion(schemaVersion);
281+
tableInfo.SetTableName(tableName);
282+
283+
QueryViews.emplace_back(std::move(tableInfo));
284+
}
285+
}
270286

271287
bool TPreparedQueryHolder::HasTempTables(TKqpTempTablesState::TConstPtr tempTablesState, bool withSessionId) const {
272288
if (!tempTablesState) {

‎ydb/core/kqp/query_data/kqp_prepared_query.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ class TPreparedQueryHolder {
137137
std::shared_ptr<const NKikimrKqp::TPreparedQuery> Proto;
138138
std::shared_ptr<TPreparedQueryAllocHolder> Alloc;
139139
TVector<TString> QueryTables;
140+
TVector<NKqpProto::TKqpTableInfo> QueryViews;
140141
std::vector<TKqpPhyTxHolder::TConstPtr> Transactions;
141142
TIntrusivePtr<TTableConstInfoMap> TableConstInfoById;
142143

@@ -180,6 +181,10 @@ class TPreparedQueryHolder {
180181
return QueryTables;
181182
}
182183

184+
const TVector<NKqpProto::TKqpTableInfo>& GetQueryViews() const {
185+
return QueryViews;
186+
}
187+
183188
const NKqpProto::TKqpPhyQuery& GetPhysicalQuery() const {
184189
return Proto->GetPhysicalQuery();
185190
}
@@ -192,6 +197,8 @@ class TPreparedQueryHolder {
192197

193198
void FillTables(const google::protobuf::RepeatedPtrField< ::NKqpProto::TKqpPhyStage>& stages);
194199

200+
void FillViews(const TVector<NKikimrKqp::TKqpTableMetadataProto>& viewsMetadata);
201+
195202
bool HasTempTables(TKqpTempTablesState::TConstPtr tempTablesState, bool withSessionId) const;
196203
};
197204

‎ydb/core/kqp/session_actor/kqp_query_state.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,6 @@ std::unique_ptr<TEvTxProxySchemeCache::TEvNavigateKeySet> TKqpQueryState::BuildN
100100
return std::make_unique<TEvTxProxySchemeCache::TEvNavigateKeySet>(navigate.Release());
101101
}
102102

103-
104103
bool TKqpQueryState::SaveAndCheckCompileResult(TEvKqp::TEvCompileResponse* ev) {
105104
CompileResult = ev->CompileResult;
106105
YQL_ENSURE(CompileResult);

‎ydb/core/kqp/session_actor/kqp_query_state.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,16 @@ class TKqpQueryState : public TNonCopyable {
205205
TableVersions.emplace(tableId, table.GetVersion());
206206
}
207207
};
208+
auto addView = [&](const NKqpProto::TKqpTableInfo& view) {
209+
NKikimr::TTableId tableId(view.GetTableId().GetOwnerId(), view.GetTableId().GetTableId());
210+
auto it = TableVersions.find(tableId);
211+
if (it != TableVersions.end()) {
212+
Y_ENSURE(it->second == view.GetSchemaVersion());
213+
} else {
214+
TableVersions.emplace(tableId, view.GetSchemaVersion());
215+
}
216+
};
217+
208218
for (const auto& stage : phyTx.GetStages()) {
209219
for (const auto& tableOp : stage.GetTableOps()) {
210220
addTable(tableOp.GetTable());
@@ -232,6 +242,9 @@ class TKqpQueryState : public TNonCopyable {
232242
addTable(table.GetId());
233243
}
234244
}
245+
for (const auto& view : PreparedQuery->GetQueryViews()) {
246+
addView(view);
247+
}
235248
}
236249

237250
bool NeedCheckTableVersions() const {

‎ydb/core/kqp/ut/view/view_ut.cpp

Lines changed: 87 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
2+
13
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
24
#include <ydb/library/yql/sql/sql.h>
35
#include <ydb/library/yql/utils/log/log.h>
@@ -8,6 +10,7 @@
810

911
using namespace NKikimr;
1012
using namespace NKikimr::NKqp;
13+
using namespace NYdb;
1114
using namespace NYdb::NTable;
1215

1316
namespace {
@@ -75,8 +78,37 @@ TDataQueryResult ExecuteDataModificationQuery(TSession& session,
7578
return result;
7679
}
7780

78-
TString GetYsonResults(TSession& session, const TString& query, const TExecDataQuerySettings& settings = {}) {
79-
return FormatResultSetYson(ExecuteDataModificationQuery(session, query, settings).GetResultSet(0));
81+
TValue GetSingleResult(const TDataQueryResult& rawResults) {
82+
auto resultSetParser = rawResults.GetResultSetParser(0);
83+
UNIT_ASSERT(resultSetParser.TryNextRow());
84+
return resultSetParser.GetValue(0);
85+
}
86+
87+
TValue GetSingleResult(TSession& session, const TString& query, const TExecDataQuerySettings& settings = {}) {
88+
return GetSingleResult(ExecuteDataModificationQuery(session, query, settings));
89+
}
90+
91+
TInstant GetTimestamp(const TValue& value) {
92+
return TValueParser(value).GetTimestamp();
93+
}
94+
95+
int GetInteger(const TValue& value) {
96+
return TValueParser(value).GetInt32();
97+
}
98+
99+
TMaybe<bool> GetFromCache(const TQueryStats& stats) {
100+
const auto& proto = NYdb::TProtoAccessor::GetProto(stats);
101+
if (!proto.Hascompilation()) {
102+
return Nothing();
103+
}
104+
return proto.Getcompilation().Getfrom_cache();
105+
}
106+
107+
void AssertFromCache(const TMaybe<TQueryStats>& stats, bool expectedValue) {
108+
UNIT_ASSERT(stats.Defined());
109+
const auto fromCache = GetFromCache(*stats);
110+
UNIT_ASSERT_C(fromCache.Defined(), stats->ToString());
111+
UNIT_ASSERT_VALUES_EQUAL_C(*fromCache, expectedValue, stats->ToString());
80112
}
81113

82114
void CompareResults(const TDataQueryResult& first, const TDataQueryResult& second) {
@@ -384,6 +416,53 @@ Y_UNIT_TEST_SUITE(TSelectFromViewTest) {
384416
ExecuteDataDefinitionQuery(session, ReadWholeFile(pathPrefix + "drop_view.sql"));
385417
}
386418
}
419+
420+
Y_UNIT_TEST(QueryCacheIsUpdated) {
421+
TKikimrRunner kikimr(TKikimrSettings().SetWithSampleTables(false));
422+
EnableViewsFeatureFlag(kikimr);
423+
auto session = kikimr.GetTableClient().CreateSession().GetValueSync().GetSession();
424+
425+
constexpr const char* viewName = "TheView";
426+
427+
const auto getCreationQuery = [&viewName](const char* innerQuery) -> TString {
428+
return std::format(R"(
429+
CREATE VIEW {} WITH (security_invoker = TRUE) AS {};
430+
)",
431+
viewName,
432+
innerQuery
433+
);
434+
};
435+
constexpr const char* firstInnerQuery = "SELECT 1";
436+
ExecuteDataDefinitionQuery(session, getCreationQuery(firstInnerQuery));
437+
438+
const TString selectFromViewQuery = std::format(R"(
439+
SELECT * FROM {};
440+
)",
441+
viewName
442+
);
443+
TExecDataQuerySettings queryExecutionSettings;
444+
queryExecutionSettings.KeepInQueryCache(true);
445+
queryExecutionSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
446+
ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
447+
// make sure the server side cache is working by calling the same query twice
448+
const auto cachedQueryRawResult = ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
449+
AssertFromCache(cachedQueryRawResult.GetStats(), true);
450+
UNIT_ASSERT_VALUES_EQUAL(GetInteger(GetSingleResult(cachedQueryRawResult)), 1);
451+
452+
// recreate the view with a different query inside
453+
ExecuteDataDefinitionQuery(session, std::format(R"(
454+
DROP VIEW {};
455+
)",
456+
viewName
457+
)
458+
);
459+
constexpr const char* secondInnerQuery = "SELECT 2";
460+
ExecuteDataDefinitionQuery(session, getCreationQuery(secondInnerQuery));
461+
462+
const auto secondCallRawResult = ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
463+
AssertFromCache(secondCallRawResult.GetStats(), false);
464+
UNIT_ASSERT_VALUES_EQUAL(GetInteger(GetSingleResult(secondCallRawResult)), 2);
465+
}
387466
}
388467

389468
Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
@@ -414,9 +493,9 @@ Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
414493
TExecDataQuerySettings queryExecutionSettings;
415494
queryExecutionSettings.KeepInQueryCache(true);
416495
const auto executeTwice = [&](const TString& query) {
417-
return TVector<TString>{
418-
GetYsonResults(session, query, queryExecutionSettings),
419-
GetYsonResults(session, query, queryExecutionSettings)
496+
return TVector<TInstant>{
497+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings)),
498+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings))
420499
};
421500
};
422501
const auto viewResults = executeTwice(selectFromViewQuery);
@@ -455,9 +534,9 @@ Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
455534
TExecDataQuerySettings queryExecutionSettings;
456535
queryExecutionSettings.KeepInQueryCache(true);
457536
const auto executeTwice = [&](const TString& query) {
458-
return TVector<TString>{
459-
GetYsonResults(session, query, queryExecutionSettings),
460-
GetYsonResults(session, query, queryExecutionSettings)
537+
return TVector<TInstant>{
538+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings)),
539+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings))
461540
};
462541
};
463542
const auto viewResults = executeTwice(selectFromViewQuery);

0 commit comments

Comments
 (0)