Skip to content

Commit 64fa639

Browse files
kungakunga
authored
24-3 Obtain UsedTableMemory from private cache stats, include historic indexes size (#6400)
1 parent 56b5d9a commit 64fa639

File tree

6 files changed

+83
-33
lines changed

6 files changed

+83
-33
lines changed

‎ydb/core/tablet_flat/flat_executor.cpp

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3556,23 +3556,23 @@ void TExecutor::Handle(NOps::TEvResult *ops, TProdCompact *msg, bool cancelled)
35563556
}
35573557

35583558
void TExecutor::UpdateUsedTabletMemory() {
3559-
UsedTabletMemory = 0;
3560-
// Estimate memory usage for internal executor structures.
3561-
UsedTabletMemory += 50 << 10; // 50kb
3562-
// Count the number of bytes exclusive to private cache.
3559+
// Estimate memory usage for internal executor structures:
3560+
UsedTabletMemory = 50 << 10; // 50kb
3561+
3562+
// Count the number of bytes kept in private cache (can't be offloaded right now):
35633563
if (PrivatePageCache) {
3564-
UsedTabletMemory += PrivatePageCache->GetStats().TotalExclusive;
3564+
UsedTabletMemory += PrivatePageCache->GetStats().TotalPinnedBody;
3565+
UsedTabletMemory += PrivatePageCache->GetStats().PinnedLoadSize;
35653566
}
3566-
// Estimate memory used by database structures.
3567+
3568+
// Estimate memory used by internal database structures:
35673569
auto &counters = Database->Counters();
35683570
UsedTabletMemory += counters.MemTableWaste;
35693571
UsedTabletMemory += counters.MemTableBytes;
3570-
UsedTabletMemory += counters.Parts.IndexBytes;
35713572
UsedTabletMemory += counters.Parts.OtherBytes;
3572-
UsedTabletMemory += counters.Parts.ByKeyBytes;
35733573
UsedTabletMemory += Stats->PacksMetaBytes;
35743574

3575-
// Add tablet memory usage.
3575+
// Add tablet memory usage:
35763576
UsedTabletMemory += Owner->GetMemoryUsage();
35773577
}
35783578

‎ydb/core/tablet_flat/flat_executor_ut.cpp

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6307,6 +6307,45 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorBTreeIndex) {
63076307
UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 330);
63086308
}
63096309

6310+
Y_UNIT_TEST(EnableLocalDBBtreeIndex_False) { // uses flat index
6311+
TMyEnvBase env;
6312+
TRowsModel rows;
6313+
6314+
auto &appData = env->GetAppData();
6315+
6316+
appData.FeatureFlags.SetEnableLocalDBBtreeIndex(false);
6317+
auto counters = MakeIntrusive<TSharedPageCacheCounters>(env->GetDynamicCounters());
6318+
int readRows = 0, failedAttempts = 0;
6319+
6320+
env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
6321+
6322+
auto policy = MakeIntrusive<TCompactionPolicy>();
6323+
policy->MinBTreeIndexNodeSize = 128;
6324+
env.SendSync(rows.MakeScheme(std::move(policy)));
6325+
6326+
env.SendSync(rows.VersionTo(TRowVersion(1, 10)).RowTo(0).MakeRows(1000, 950));
6327+
env.SendSync(rows.VersionTo(TRowVersion(2, 20)).RowTo(0).MakeRows(1000, 950));
6328+
6329+
env.SendSync(new NFake::TEvCompact(TRowsModel::TableId));
6330+
env.WaitFor<NFake::TEvCompacted>();
6331+
6332+
// all pages are always kept in shared cache
6333+
UNIT_ASSERT_VALUES_EQUAL(counters->ActivePages->Val(), 290);
6334+
6335+
env.SendSync(new NFake::TEvExecute{ new TTxFullScan(readRows, failedAttempts) });
6336+
UNIT_ASSERT_VALUES_EQUAL(readRows, 1000);
6337+
UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 0);
6338+
6339+
// restart tablet
6340+
env.SendSync(new TEvents::TEvPoison, false, true);
6341+
env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
6342+
6343+
// after restart we have no pages in private cache
6344+
env.SendSync(new NFake::TEvExecute{ new TTxFullScan(readRows, failedAttempts) }, true);
6345+
UNIT_ASSERT_VALUES_EQUAL(readRows, 1000);
6346+
UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 288);
6347+
}
6348+
63106349
Y_UNIT_TEST(EnableLocalDBBtreeIndex_True_EnableLocalDBFlatIndex_False) { // uses b-tree index
63116350
TMyEnvBase env;
63126351
TRowsModel rows;

‎ydb/core/tablet_flat/flat_part_loader.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,17 +210,22 @@ TAutoPtr<NPageCollection::TFetch> TLoader::StageCreatePartView() noexcept
210210
TEpoch epoch = Epoch != TEpoch::Max() ? Epoch : TEpoch(Root.GetEpoch());
211211

212212
// TODO: put index size to stat?
213-
// TODO: include history indexes bytes
214213
size_t indexesRawSize = 0;
215214
if (BTreeGroupIndexes) {
216215
for (const auto &meta : BTreeGroupIndexes) {
217216
indexesRawSize += meta.IndexSize;
218217
}
218+
for (const auto &meta : BTreeHistoricIndexes) {
219+
indexesRawSize += meta.IndexSize;
220+
}
219221
// Note: although we also have flat index, it shouldn't be loaded; so let's not count it here
220222
} else {
221223
for (auto indexPage : FlatGroupIndexes) {
222224
indexesRawSize += Packs[0]->GetPageSize(indexPage);
223225
}
226+
for (auto indexPage : FlatHistoricIndexes) {
227+
indexesRawSize += Packs[0]->GetPageSize(indexPage);
228+
}
224229
}
225230

226231
auto *partStore = new TPartStore(

‎ydb/core/tablet_flat/test/libs/table/test_store.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ namespace NTest {
2424

2525
struct TEggs {
2626
bool Rooted;
27-
TVector<TPageId> GroupIndexes;
28-
TVector<TPageId> HistoricIndexes;
27+
TVector<TPageId> FlatGroupIndexes;
28+
TVector<TPageId> FlatHistoricIndexes;
2929
TVector<NPage::TBtreeIndexMeta> BTreeGroupIndexes;
3030
TVector<NPage::TBtreeIndexMeta> BTreeHistoricIndexes;
3131
TData *Scheme;

‎ydb/core/tablet_flat/test/libs/table/test_writer.h

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,14 @@ namespace NTest {
7777
for (const auto &meta : eggs.BTreeGroupIndexes) {
7878
indexesRawSize += meta.IndexSize;
7979
}
80+
for (const auto &meta : eggs.BTreeHistoricIndexes) {
81+
indexesRawSize += meta.IndexSize;
82+
}
8083
} else {
81-
for (auto indexPage : eggs.GroupIndexes) {
84+
for (auto indexPage : eggs.FlatGroupIndexes) {
85+
indexesRawSize += Store->GetPageSize(0, indexPage);
86+
}
87+
for (auto indexPage : eggs.FlatHistoricIndexes) {
8288
indexesRawSize += Store->GetPageSize(0, indexPage);
8389
}
8490
}
@@ -90,7 +96,7 @@ namespace NTest {
9096
{
9197
epoch,
9298
TPartScheme::Parse(*eggs.Scheme, eggs.Rooted),
93-
{ eggs.GroupIndexes, eggs.HistoricIndexes, eggs.BTreeGroupIndexes, eggs.BTreeHistoricIndexes },
99+
{ eggs.FlatGroupIndexes, eggs.FlatHistoricIndexes, eggs.BTreeGroupIndexes, eggs.BTreeHistoricIndexes },
94100
eggs.Blobs ? new TExtBlobs(*eggs.Blobs, { }) : nullptr,
95101
eggs.ByKey ? new TBloom(*eggs.ByKey) : nullptr,
96102
eggs.Large ? new TFrames(*eggs.Large) : nullptr,

‎ydb/core/tablet_flat/ut/ut_stat.cpp

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -143,14 +143,14 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
143143
Y_UNIT_TEST(Single_History)
144144
{
145145
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
146-
CheckMixedIndex(*subset, 24000, 3547100, 31242);
146+
CheckMixedIndex(*subset, 24000, 3547100, 49916);
147147
}
148148

149149
Y_UNIT_TEST(Single_History_Slices)
150150
{
151151
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
152152
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
153-
CheckMixedIndex(*subset, 9582, 1425198, 31242);
153+
CheckMixedIndex(*subset, 9582, 1425198, 49916);
154154
}
155155

156156
Y_UNIT_TEST(Single_Groups)
@@ -169,14 +169,14 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
169169
Y_UNIT_TEST(Single_Groups_History)
170170
{
171171
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
172-
CheckMixedIndex(*subset, 24000, 4054050, 18810);
172+
CheckMixedIndex(*subset, 24000, 4054050, 29361);
173173
}
174174

175175
Y_UNIT_TEST(Single_Groups_History_Slices)
176176
{
177177
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
178178
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
179-
CheckMixedIndex(*subset, 13570, 2277890, 18810);
179+
CheckMixedIndex(*subset, 13570, 2277890, 29361);
180180
}
181181

182182
Y_UNIT_TEST(Mixed)
@@ -194,7 +194,7 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
194194
Y_UNIT_TEST(Mixed_Groups_History)
195195
{
196196
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3);
197-
CheckMixedIndex(*subset, 24000, 4054270, 19152);
197+
CheckMixedIndex(*subset, 24000, 4054270, 29970);
198198
}
199199

200200
Y_UNIT_TEST(Serial)
@@ -215,7 +215,7 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
215215
{
216216
TMixerSeq mixer(4, Mass1.Saved.Size());
217217
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, mixer, 0.3);
218-
CheckMixedIndex(*subset, 24000, 4054290, 19168);
218+
CheckMixedIndex(*subset, 24000, 4054290, 30013);
219219
}
220220
}
221221

@@ -239,14 +239,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
239239
Y_UNIT_TEST(Single_History)
240240
{
241241
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
242-
CheckMixedIndex(*subset, 24000, 3547100, 61162);
242+
CheckMixedIndex(*subset, 24000, 3547100, 81694);
243243
}
244244

245245
Y_UNIT_TEST(Single_History_Slices)
246246
{
247247
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
248248
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
249-
CheckMixedIndex(*subset, 9582, 1425198, 61162);
249+
CheckMixedIndex(*subset, 9582, 1425198, 81694);
250250
}
251251

252252
Y_UNIT_TEST(Single_Groups)
@@ -265,14 +265,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
265265
Y_UNIT_TEST(Single_Groups_History)
266266
{
267267
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
268-
CheckMixedIndex(*subset, 24000, 4054050, 34837);
268+
CheckMixedIndex(*subset, 24000, 4054050, 46562);
269269
}
270270

271271
Y_UNIT_TEST(Single_Groups_History_Slices)
272272
{
273273
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
274274
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
275-
CheckMixedIndex(*subset, 13570, 2277890, 34837);
275+
CheckMixedIndex(*subset, 13570, 2277890, 46562);
276276
}
277277

278278
Y_UNIT_TEST(Mixed)
@@ -290,7 +290,7 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
290290
Y_UNIT_TEST(Mixed_Groups_History)
291291
{
292292
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3);
293-
CheckMixedIndex(*subset, 24000, 4054270, 34579);
293+
CheckMixedIndex(*subset, 24000, 4054270, 46543);
294294
}
295295

296296
Y_UNIT_TEST(Serial)
@@ -311,7 +311,7 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
311311
{
312312
TMixerSeq mixer(4, Mass1.Saved.Size());
313313
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, mixer, 0.3);
314-
CheckMixedIndex(*subset, 24000, 4054290, 34652);
314+
CheckMixedIndex(*subset, 24000, 4054290, 46640);
315315
}
316316

317317
Y_UNIT_TEST(Single_LowResolution)
@@ -343,14 +343,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
343343
Y_UNIT_TEST(Single_Groups_History_LowResolution)
344344
{
345345
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), true, WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
346-
CheckMixedIndex(*subset, 24000, 4054050, 48540, 5310, 531050);
346+
CheckMixedIndex(*subset, 24000, 4054050, 64742, 5310, 531050);
347347
}
348348

349349
Y_UNIT_TEST(Single_Groups_History_Slices_LowResolution)
350350
{
351351
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), true, WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
352352
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
353-
CheckMixedIndex(*subset, 13570, 2234982 /* ~2277890 */, 48540, 5310, 531050);
353+
CheckMixedIndex(*subset, 13570, 2234982 /* ~2277890 */, 64742, 5310, 531050);
354354
}
355355
}
356356

@@ -374,14 +374,14 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) {
374374
Y_UNIT_TEST(Single_History)
375375
{
376376
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
377-
CheckBTreeIndex(*subset, 24000, 3547100, 61162);
377+
CheckBTreeIndex(*subset, 24000, 3547100, 81694);
378378
}
379379

380380
Y_UNIT_TEST(Single_History_Slices)
381381
{
382382
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
383383
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
384-
CheckBTreeIndex(*subset, 9582, 1425282, 61162);
384+
CheckBTreeIndex(*subset, 9582, 1425282, 81694);
385385
}
386386

387387
Y_UNIT_TEST(Single_Groups)
@@ -400,14 +400,14 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) {
400400
Y_UNIT_TEST(Single_Groups_History)
401401
{
402402
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
403-
CheckBTreeIndex(*subset, 24000, 4054050, 34837);
403+
CheckBTreeIndex(*subset, 24000, 4054050, 46562);
404404
}
405405

406406
Y_UNIT_TEST(Single_Groups_History_Slices)
407407
{
408408
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
409409
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
410-
CheckBTreeIndex(*subset, 13570, 2273213, 34837);
410+
CheckBTreeIndex(*subset, 13570, 2273213, 46562);
411411
}
412412

413413
Y_UNIT_TEST(Mixed)
@@ -425,7 +425,7 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) {
425425
Y_UNIT_TEST(Mixed_Groups_History)
426426
{
427427
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3);
428-
CheckBTreeIndex(*subset, 24000, 4054270, 34579);
428+
CheckBTreeIndex(*subset, 24000, 4054270, 46543);
429429
}
430430
}
431431

0 commit comments

Comments
 (0)