Skip to content

Commit 0041818

Browse files
snaurysnaury
authored
Merge f6685ae into 6067a04 
2 parents 6067a04 + f6685ae commit 0041818

File tree

10 files changed

+155
-11
lines changed

10 files changed

+155
-11
lines changed

‎ydb/core/tablet_flat/flat_dbase_apply.cpp

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,14 @@ bool TSchemeModifier::Apply(const TAlterRecord &delta)
9797
ui32 large = delta.HasLarge() ? delta.GetLarge() : family.Large;
9898

9999
Y_ABORT_UNLESS(ui32(cache) <= 2, "Invalid pages cache policy value");
100-
changes |= ChangeTableSetting(table, family.Cache, cache);
100+
if (family.Cache != cache) {
101+
PreserveTable(table);
102+
family.Cache = cache;
103+
changes |= true;
104+
if (cache == ECache::Ever) {
105+
AddNeedToCacheTable(table);
106+
}
107+
}
101108
changes |= ChangeTableSetting(table, family.Codec, codec);
102109
changes |= ChangeTableSetting(table, family.Small, small);
103110
changes |= ChangeTableSetting(table, family.Large, large);
@@ -405,6 +412,15 @@ bool TSchemeModifier::SetCompactionPolicy(ui32 tid, const NKikimrCompaction::TCo
405412
return true;
406413
}
407414

415+
void TSchemeModifier::AddNeedToCacheTable(ui32 tid) noexcept
416+
{
417+
if (Scheme.NeedToCacheTables.insert(tid).second) {
418+
if (RollbackState) {
419+
RollbackState->NeedToCacheTables.insert(tid);
420+
}
421+
}
422+
}
423+
408424
void TSchemeModifier::PreserveTable(ui32 tid) noexcept
409425
{
410426
if (RollbackState && !RollbackState->Tables.contains(tid)) {

‎ydb/core/tablet_flat/flat_dbase_apply.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ namespace NTable {
8383
return false;
8484
}
8585

86+
void AddNeedToCacheTable(ui32 tid) noexcept;
87+
8688
void PreserveTable(ui32 tid) noexcept;
8789
void PreserveExecutor() noexcept;
8890
void PreserveRedo() noexcept;

‎ydb/core/tablet_flat/flat_dbase_naked.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,7 @@ namespace NTable {
361361
SchemeRollbackState.Tables.clear();
362362
SchemeRollbackState.Executor.reset();
363363
SchemeRollbackState.Redo.reset();
364+
SchemeRollbackState.NeedToCacheTables.clear();
364365
}
365366

366367
void RollbackTransaction() noexcept
@@ -401,6 +402,10 @@ namespace NTable {
401402
// Note: we assume schema rollback is very rare,
402403
// so it doesn't have to be efficient
403404
TScheme& scheme = *Scheme;
405+
for (ui32 tid : SchemeRollbackState.NeedToCacheTables) {
406+
scheme.NeedToCacheTables.erase(tid);
407+
}
408+
SchemeRollbackState.NeedToCacheTables.clear();
404409
if (SchemeRollbackState.Redo) {
405410
scheme.Redo = *SchemeRollbackState.Redo;
406411
SchemeRollbackState.Redo.reset();

‎ydb/core/tablet_flat/flat_dbase_scheme.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,9 @@ class TScheme {
171171
THashMap<TString, ui32> TableNames;
172172
TExecutorInfo Executor;
173173
TRedo Redo;
174+
175+
// A set of tables that have their in-memory state enabled
176+
mutable THashSet<ui32> NeedToCacheTables;
174177
};
175178

176179
/**
@@ -186,6 +189,8 @@ struct TSchemeRollbackState {
186189
std::optional<TScheme::TExecutorInfo> Executor;
187190
// Previous redo settings if modified
188191
std::optional<TScheme::TRedo> Redo;
192+
// Newly added NeedToCacheTables entries
193+
THashSet<ui32> NeedToCacheTables;
189194
};
190195

191196
/**

‎ydb/core/tablet_flat/flat_executor.cpp

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,8 @@ void TExecutor::ActivateFollower(const TActorContext &ctx) {
408408
RecreatePageCollectionsCache();
409409
ReflectSchemeSettings();
410410

411+
RequestInMemPagesForDatabase();
412+
411413
Become(&TThis::StateFollower);
412414
Stats->IsActive = true;
413415
Stats->FollowerId = FollowerId;
@@ -648,16 +650,28 @@ void TExecutor::TranslateCacheTouchesToSharedCache() {
648650
}
649651

650652
void TExecutor::RequestInMemPagesForDatabase() {
653+
const auto& scheme = Scheme();
654+
for (auto& pr : Scheme().Tables) {
655+
scheme.NeedToCacheTables.insert(pr.first);
656+
}
657+
RequestInMemPagesForNeedToCacheTables();
658+
}
659+
660+
void TExecutor::RequestInMemPagesForNeedToCacheTables() {
651661
const auto &scheme = Scheme();
652-
for (auto &sxpair : scheme.Tables) {
653-
auto stickyColumns = GetStickyColumns(sxpair.first);
662+
for (ui32 tid : scheme.NeedToCacheTables) {
663+
if (!scheme.Tables.contains(tid)) {
664+
continue;
665+
}
666+
auto stickyColumns = GetStickyColumns(tid);
654667
if (stickyColumns) {
655-
auto subset = Database->Subset(sxpair.first, NTable::TEpoch::Max(), { } , { });
668+
auto subset = Database->Subset(tid, NTable::TEpoch::Max(), { } , { });
656669

657670
for (auto &partView: subset->Flatten)
658-
RequestInMemPagesForPartStore(sxpair.first, partView, stickyColumns);
671+
RequestInMemPagesForPartStore(tid, partView, stickyColumns);
659672
}
660673
}
674+
scheme.NeedToCacheTables.clear();
661675
}
662676

663677
TExecutorCaches TExecutor::CleanupState() {
@@ -968,6 +982,7 @@ void TExecutor::ApplyFollowerUpdate(THolder<TEvTablet::TFUpdateBody> update) {
968982
if (schemeUpdate) {
969983
ReadResourceProfile();
970984
ReflectSchemeSettings();
985+
RequestInMemPagesForNeedToCacheTables();
971986
Owner->OnFollowerSchemaUpdated();
972987
}
973988

@@ -2065,6 +2080,7 @@ void TExecutor::CommitTransactionLog(TAutoPtr<TSeat> seat, TPageCollectionTxEnv
20652080

20662081
ReadResourceProfile();
20672082
ReflectSchemeSettings();
2083+
RequestInMemPagesForNeedToCacheTables();
20682084

20692085
// For every table that changed strategy we need to generate a
20702086
// special part switch that notifies bootlogic about new strategy

‎ydb/core/tablet_flat/flat_executor.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -514,6 +514,7 @@ class TExecutor
514514

515515
void TranslateCacheTouchesToSharedCache();
516516
void RequestInMemPagesForDatabase();
517+
void RequestInMemPagesForNeedToCacheTables();
517518
void RequestInMemPagesForPartStore(ui32 tableId, const NTable::TPartView &partView, const THashSet<NTable::TTag> &stickyColumns);
518519
THashSet<NTable::TTag> GetStickyColumns(ui32 tableId);
519520
void RequestFromSharedCache(TAutoPtr<NPageCollection::TFetch> fetch,

‎ydb/core/ydb_convert/column_families.h

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -183,13 +183,15 @@ namespace NKikimr {
183183
case Ydb::FeatureFlag::STATUS_UNSPECIFIED:
184184
break;
185185
case Ydb::FeatureFlag::ENABLED:
186-
*code = Ydb::StatusIds::BAD_REQUEST;
187-
*error = TStringBuilder()
188-
<< "Setting keep_in_memory to ENABLED is not supported in column family '"
189-
<< familySettings.name() << "'";
190-
return false;
186+
if (!AppData()->FeatureFlags.GetEnablePublicApiKeepInMemory()) {
187+
*code = Ydb::StatusIds::BAD_REQUEST;
188+
*error = "Setting keep_in_memory to ENABLED is not allowed";
189+
return false;
190+
}
191+
family->SetColumnCache(NKikimrSchemeOp::ColumnCacheEver);
192+
break;
191193
case Ydb::FeatureFlag::DISABLED:
192-
family->ClearColumnCache();
194+
family->SetColumnCache(NKikimrSchemeOp::ColumnCacheNone);
193195
break;
194196
default:
195197
*code = Ydb::StatusIds::BAD_REQUEST;

‎ydb/public/sdk/cpp/client/ydb_table/table.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1115,6 +1115,11 @@ TColumnFamilyBuilder& TColumnFamilyBuilder::SetCompression(EColumnFamilyCompress
11151115
return *this;
11161116
}
11171117

1118+
TColumnFamilyBuilder& TColumnFamilyBuilder::SetKeepInMemory(bool enabled) {
1119+
Impl_->Proto.set_keep_in_memory(enabled ? Ydb::FeatureFlag::ENABLED : Ydb::FeatureFlag::DISABLED);
1120+
return *this;
1121+
}
1122+
11181123
TColumnFamilyDescription TColumnFamilyBuilder::Build() const {
11191124
return TColumnFamilyDescription(Impl_->Proto);
11201125
}

‎ydb/public/sdk/cpp/client/ydb_table/table.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -806,6 +806,7 @@ class TColumnFamilyBuilder {
806806

807807
TColumnFamilyBuilder& SetData(const TString& media);
808808
TColumnFamilyBuilder& SetCompression(EColumnFamilyCompression compression);
809+
TColumnFamilyBuilder& SetKeepInMemory(bool enabled);
809810

810811
TColumnFamilyDescription Build() const;
811812

@@ -868,6 +869,11 @@ class TTableColumnFamilyBuilder {
868869
return *this;
869870
}
870871

872+
TTableColumnFamilyBuilder& SetKeepInMemory(bool enabled) {
873+
Builder_.SetKeepInMemory(enabled);
874+
return *this;
875+
}
876+
871877
TTableBuilder& EndColumnFamily();
872878

873879
private:
@@ -1486,6 +1492,11 @@ class TAlterColumnFamilyBuilder {
14861492
return *this;
14871493
}
14881494

1495+
TAlterColumnFamilyBuilder& SetKeepInMemory(bool enabled) {
1496+
Builder_.SetKeepInMemory(enabled);
1497+
return *this;
1498+
}
1499+
14891500
TAlterTableSettings& EndAddColumnFamily();
14901501
TAlterTableSettings& EndAlterColumnFamily();
14911502

‎ydb/services/ydb/ydb_ut.cpp

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -824,6 +824,87 @@ Y_UNIT_TEST_SUITE(TGRpcNewClient) {
824824
client.CreateSession().Apply(createSessionHandler).Wait();
825825
UNIT_ASSERT(done);
826826
}
827+
828+
Y_UNIT_TEST(InMemoryTables) {
829+
TKikimrWithGrpcAndRootSchemaNoSystemViews server;
830+
server.Server_->GetRuntime()->GetAppData().FeatureFlags.SetEnablePublicApiKeepInMemory(true);
831+
832+
ui16 grpc = server.GetPort();
833+
TString location = TStringBuilder() << "localhost:" << grpc;
834+
835+
auto connection = NYdb::TDriver(
836+
TDriverConfig()
837+
.SetEndpoint(location));
838+
839+
auto client = NYdb::NTable::TTableClient(connection);
840+
auto createSessionResult = client.CreateSession().ExtractValueSync();
841+
UNIT_ASSERT(!createSessionResult.IsTransportError());
842+
auto session = createSessionResult.GetSession();
843+
844+
auto createTableResult = session.CreateTable("/Root/Table", client.GetTableBuilder()
845+
.AddNullableColumn("Key", EPrimitiveType::Int32)
846+
.AddNullableColumn("Value", EPrimitiveType::String)
847+
.SetPrimaryKeyColumn("Key")
848+
// Note: only needed because this test doesn't initial table profiles
849+
.BeginStorageSettings()
850+
.SetTabletCommitLog0("ssd")
851+
.SetTabletCommitLog1("ssd")
852+
.EndStorageSettings()
853+
.BeginColumnFamily("default")
854+
.SetData("ssd")
855+
.SetKeepInMemory(true)
856+
.EndColumnFamily()
857+
.Build()).ExtractValueSync();
858+
UNIT_ASSERT_C(createTableResult.IsSuccess(), (NYdb::TStatus&)createTableResult);
859+
860+
{
861+
auto describeTableResult = session.DescribeTable("/Root/Table").ExtractValueSync();
862+
UNIT_ASSERT_C(describeTableResult.IsSuccess(), (NYdb::TStatus&)describeTableResult);
863+
auto desc = describeTableResult.GetTableDescription();
864+
auto families = desc.GetColumnFamilies();
865+
UNIT_ASSERT_VALUES_EQUAL(families.size(), 1u);
866+
auto family = families.at(0);
867+
UNIT_ASSERT_VALUES_EQUAL(family.GetKeepInMemory(), true);
868+
}
869+
870+
{
871+
auto alterTableResult = session.AlterTable("/Root/Table", NYdb::NTable::TAlterTableSettings()
872+
.BeginAlterColumnFamily("default")
873+
.SetKeepInMemory(false)
874+
.EndAlterColumnFamily()).ExtractValueSync();
875+
UNIT_ASSERT_C(alterTableResult.IsSuccess(), (NYdb::TStatus&)alterTableResult);
876+
}
877+
878+
{
879+
auto describeTableResult = session.DescribeTable("/Root/Table").ExtractValueSync();
880+
UNIT_ASSERT_C(describeTableResult.IsSuccess(), (NYdb::TStatus&)describeTableResult);
881+
auto desc = describeTableResult.GetTableDescription();
882+
auto families = desc.GetColumnFamilies();
883+
UNIT_ASSERT_VALUES_EQUAL(families.size(), 1u);
884+
auto family = families.at(0);
885+
// Note: server cannot currently distinguish between implicitly
886+
// unset and explicitly disabled, so it returns the former.
887+
UNIT_ASSERT_VALUES_EQUAL(family.GetKeepInMemory(), Nothing());
888+
}
889+
890+
{
891+
auto alterTableResult = session.AlterTable("/Root/Table", NYdb::NTable::TAlterTableSettings()
892+
.BeginAlterColumnFamily("default")
893+
.SetKeepInMemory(true)
894+
.EndAlterColumnFamily()).ExtractValueSync();
895+
UNIT_ASSERT_C(alterTableResult.IsSuccess(), (NYdb::TStatus&)alterTableResult);
896+
}
897+
898+
{
899+
auto describeTableResult = session.DescribeTable("/Root/Table").ExtractValueSync();
900+
UNIT_ASSERT_C(describeTableResult.IsSuccess(), (NYdb::TStatus&)describeTableResult);
901+
auto desc = describeTableResult.GetTableDescription();
902+
auto families = desc.GetColumnFamilies();
903+
UNIT_ASSERT_VALUES_EQUAL(families.size(), 1u);
904+
auto family = families.at(0);
905+
UNIT_ASSERT_VALUES_EQUAL(family.GetKeepInMemory(), true);
906+
}
907+
}
827908
}
828909

829910
static TString CreateSession(std::shared_ptr<grpc::Channel> channel) {

0 commit comments

Comments
 (0)