Skip to content

Commit 02df54f

Browse files
jepett0jepett0
committed
Update query cache if the schema version of the views used in this query was updated
Add views, whose metadata was loaded during the compilation of the query, info to the prepared query proto and use the list to check (with SchemeCache) if the schema version of the view was updated and a recompilation is needed.
1 parent 28fbe05 commit 02df54f

File tree

5 files changed

+112
-8
lines changed

5 files changed

+112
-8
lines changed

‎ydb/core/kqp/provider/yql_kikimr_datasource.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,17 @@ class TKiSourceLoadTableMetadataTransformer : public TGraphTransformerBase {
351351
if (!AddCluster(table, res, input, ctx)) {
352352
return TStatus::Error;
353353
}
354+
355+
if (res.Metadata->Kind == EKikimrTableKind::View) {
356+
const auto& viewMetadata = *res.Metadata;
357+
const auto& preparingQuery = SessionCtx->Query().PreparingQuery;
358+
YQL_ENSURE(preparingQuery);
359+
auto* viewInfo = preparingQuery->MutablePhysicalQuery()->MutableViewsInfo()->Add();
360+
auto* pathId = viewInfo->MutableTableId();
361+
pathId->SetOwnerId(viewMetadata.PathId.OwnerId());
362+
pathId->SetTableId(viewMetadata.PathId.TableId());
363+
viewInfo->SetSchemaVersion(viewMetadata.SchemaVersion);
364+
}
354365
} else {
355366
TIssueScopeGuard issueScope(ctx.IssueManager, [input, &table, &ctx]() {
356367
return MakeIntrusive<TIssue>(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder()

‎ydb/core/kqp/session_actor/kqp_query_state.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,13 +71,24 @@ bool TKqpQueryState::EnsureTableVersions(const TEvTxProxySchemeCache::TEvNavigat
7171
return true;
7272
}
7373

74+
void TKqpQueryState::FillViews(const google::protobuf::RepeatedPtrField< ::NKqpProto::TKqpTableInfo>& views) {
75+
for (const auto& view : views) {
76+
const auto& pathId = view.GetTableId();
77+
const auto schemaVersion = view.GetSchemaVersion();
78+
auto [it, isInserted] = TableVersions.emplace(TTableId(pathId.GetOwnerId(), pathId.GetTableId()), schemaVersion);
79+
if (!isInserted) {
80+
Y_ENSURE(it->second == schemaVersion);
81+
}
82+
}
83+
}
7484

7585
std::unique_ptr<TEvTxProxySchemeCache::TEvNavigateKeySet> TKqpQueryState::BuildNavigateKeySet() {
7686
TableVersions.clear();
7787

7888
for (const auto& tx : PreparedQuery->GetPhysicalQuery().GetTransactions()) {
7989
FillTables(tx);
8090
}
91+
FillViews(PreparedQuery->GetPhysicalQuery().GetViewsInfo());
8192

8293
auto navigate = MakeHolder<NSchemeCache::TSchemeCacheNavigate>();
8394
navigate->DatabaseName = Database;

‎ydb/core/kqp/session_actor/kqp_query_state.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,8 @@ class TKqpQueryState : public TNonCopyable {
234234
}
235235
}
236236

237+
void FillViews(const google::protobuf::RepeatedPtrField< ::NKqpProto::TKqpTableInfo>& views);
238+
237239
bool NeedCheckTableVersions() const {
238240
return CompileStats.FromCache;
239241
}

‎ydb/core/kqp/ut/view/view_ut.cpp

Lines changed: 86 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
22
#include <ydb/library/yql/sql/sql.h>
33
#include <ydb/library/yql/utils/log/log.h>
4+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
45

56
#include <util/folder/filelist.h>
67

78
#include <format>
89

910
using namespace NKikimr;
1011
using namespace NKikimr::NKqp;
12+
using namespace NYdb;
1113
using namespace NYdb::NTable;
1214

1315
namespace {
@@ -75,8 +77,37 @@ TDataQueryResult ExecuteDataModificationQuery(TSession& session,
7577
return result;
7678
}
7779

78-
TString GetYsonResults(TSession& session, const TString& query, const TExecDataQuerySettings& settings = {}) {
79-
return FormatResultSetYson(ExecuteDataModificationQuery(session, query, settings).GetResultSet(0));
80+
TValue GetSingleResult(const TDataQueryResult& rawResults) {
81+
auto resultSetParser = rawResults.GetResultSetParser(0);
82+
UNIT_ASSERT(resultSetParser.TryNextRow());
83+
return resultSetParser.GetValue(0);
84+
}
85+
86+
TValue GetSingleResult(TSession& session, const TString& query, const TExecDataQuerySettings& settings = {}) {
87+
return GetSingleResult(ExecuteDataModificationQuery(session, query, settings));
88+
}
89+
90+
TInstant GetTimestamp(const TValue& value) {
91+
return TValueParser(value).GetTimestamp();
92+
}
93+
94+
int GetInteger(const TValue& value) {
95+
return TValueParser(value).GetInt32();
96+
}
97+
98+
TMaybe<bool> GetFromCacheStat(const TQueryStats& stats) {
99+
const auto& proto = TProtoAccessor::GetProto(stats);
100+
if (!proto.Hascompilation()) {
101+
return Nothing();
102+
}
103+
return proto.Getcompilation().Getfrom_cache();
104+
}
105+
106+
void AssertFromCache(const TMaybe<TQueryStats>& stats, bool expectedValue) {
107+
UNIT_ASSERT(stats.Defined());
108+
const auto isFromCache = GetFromCacheStat(*stats);
109+
UNIT_ASSERT_C(isFromCache.Defined(), stats->ToString());
110+
UNIT_ASSERT_VALUES_EQUAL_C(*isFromCache, expectedValue, stats->ToString());
80111
}
81112

82113
void CompareResults(const TDataQueryResult& first, const TDataQueryResult& second) {
@@ -384,6 +415,53 @@ Y_UNIT_TEST_SUITE(TSelectFromViewTest) {
384415
ExecuteDataDefinitionQuery(session, ReadWholeFile(pathPrefix + "drop_view.sql"));
385416
}
386417
}
418+
419+
Y_UNIT_TEST(QueryCacheIsUpdated) {
420+
TKikimrRunner kikimr(TKikimrSettings().SetWithSampleTables(false));
421+
EnableViewsFeatureFlag(kikimr);
422+
auto session = kikimr.GetTableClient().CreateSession().GetValueSync().GetSession();
423+
424+
constexpr const char* viewName = "TheView";
425+
426+
const auto getCreationQuery = [&viewName](const char* innerQuery) -> TString {
427+
return std::format(R"(
428+
CREATE VIEW {} WITH (security_invoker = TRUE) AS {};
429+
)",
430+
viewName,
431+
innerQuery
432+
);
433+
};
434+
constexpr const char* firstInnerQuery = "SELECT 1";
435+
ExecuteDataDefinitionQuery(session, getCreationQuery(firstInnerQuery));
436+
437+
const TString selectFromViewQuery = std::format(R"(
438+
SELECT * FROM {};
439+
)",
440+
viewName
441+
);
442+
TExecDataQuerySettings queryExecutionSettings;
443+
queryExecutionSettings.KeepInQueryCache(true);
444+
queryExecutionSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
445+
ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
446+
// make sure the server side cache is working by calling the same query twice
447+
const auto cachedQueryRawResult = ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
448+
AssertFromCache(cachedQueryRawResult.GetStats(), true);
449+
UNIT_ASSERT_VALUES_EQUAL(GetInteger(GetSingleResult(cachedQueryRawResult)), 1);
450+
451+
// recreate the view with a different query inside
452+
ExecuteDataDefinitionQuery(session, std::format(R"(
453+
DROP VIEW {};
454+
)",
455+
viewName
456+
)
457+
);
458+
constexpr const char* secondInnerQuery = "SELECT 2";
459+
ExecuteDataDefinitionQuery(session, getCreationQuery(secondInnerQuery));
460+
461+
const auto secondCallRawResult = ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
462+
AssertFromCache(secondCallRawResult.GetStats(), false);
463+
UNIT_ASSERT_VALUES_EQUAL(GetInteger(GetSingleResult(secondCallRawResult)), 2);
464+
}
387465
}
388466

389467
Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
@@ -414,9 +492,9 @@ Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
414492
TExecDataQuerySettings queryExecutionSettings;
415493
queryExecutionSettings.KeepInQueryCache(true);
416494
const auto executeTwice = [&](const TString& query) {
417-
return TVector<TString>{
418-
GetYsonResults(session, query, queryExecutionSettings),
419-
GetYsonResults(session, query, queryExecutionSettings)
495+
return TVector<TInstant>{
496+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings)),
497+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings))
420498
};
421499
};
422500
const auto viewResults = executeTwice(selectFromViewQuery);
@@ -455,9 +533,9 @@ Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
455533
TExecDataQuerySettings queryExecutionSettings;
456534
queryExecutionSettings.KeepInQueryCache(true);
457535
const auto executeTwice = [&](const TString& query) {
458-
return TVector<TString>{
459-
GetYsonResults(session, query, queryExecutionSettings),
460-
GetYsonResults(session, query, queryExecutionSettings)
536+
return TVector<TInstant>{
537+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings)),
538+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings))
461539
};
462540
};
463541
const auto viewResults = executeTwice(selectFromViewQuery);

‎ydb/core/protos/kqp_physical.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -507,4 +507,6 @@ message TKqpPhyQuery {
507507
bool HasUncommittedChangesRead = 9;
508508

509509
string QueryDiagnostics = 10;
510+
511+
repeated TKqpTableInfo ViewsInfo = 11;
510512
}

0 commit comments

Comments
 (0)