Skip to content

Commit 87d8083

Browse files
kungakunga
authored
Report Bloom filter size (#7729)
1 parent d569f4b commit 87d8083

16 files changed

+46
-29
lines changed

‎ydb/core/protos/sys_view.proto

+2
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ message TPartitionStats {
4444
optional uint64 TxRejectedBySpace = 19;
4545

4646
optional TTtlStats TtlStats = 20;
47+
48+
optional uint64 ByKeyFilterSize = 21;
4749
}
4850

4951
message TPartitionStatsResult {

‎ydb/core/protos/table_stats.proto

+2
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,6 @@ message TTableStats {
6666
repeated TChannelStats Channels = 30;
6767

6868
optional TStoragePoolsStats StoragePools = 31;
69+
70+
optional uint64 ByKeyFilterSize = 32;
6971
}

‎ydb/core/tablet_flat/flat_stat_table.h

+3
Original file line numberDiff line numberDiff line change
@@ -102,13 +102,15 @@ struct TStats {
102102
ui64 RowCount = 0;
103103
TChanneledDataSize DataSize = { };
104104
TChanneledDataSize IndexSize = { };
105+
ui64 ByKeyFilterSize = 0;
105106
THistogram RowCountHistogram;
106107
THistogram DataSizeHistogram;
107108

108109
void Clear() {
109110
RowCount = 0;
110111
DataSize = { };
111112
IndexSize = { };
113+
ByKeyFilterSize = 0;
112114
RowCountHistogram.clear();
113115
DataSizeHistogram.clear();
114116
}
@@ -117,6 +119,7 @@ struct TStats {
117119
std::swap(RowCount, other.RowCount);
118120
std::swap(DataSize, other.DataSize);
119121
std::swap(IndexSize, other.IndexSize);
122+
std::swap(ByKeyFilterSize, other.ByKeyFilterSize);
120123
RowCountHistogram.swap(other.RowCountHistogram);
121124
DataSizeHistogram.swap(other.DataSizeHistogram);
122125
}

‎ydb/core/tablet_flat/flat_stat_table_btree_index.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ bool BuildStatsBTreeIndex(const TSubset& subset, TStats& stats, ui32 histogramBu
200200
bool ready = true;
201201
for (const auto& part : subset.Flatten) {
202202
stats.IndexSize.Add(part->IndexesRawSize, part->Label.Channel());
203+
stats.ByKeyFilterSize += part->ByKey ? part->ByKey->Raw.size() : 0;
203204
ready &= AddDataSize(part, stats, env, yieldHandler);
204205
}
205206

‎ydb/core/tablet_flat/flat_stat_table_mixed_index.h

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ inline bool BuildStatsMixedIndex(const TSubset& subset, TStats& stats, ui64 rowC
2222
bool started = true;
2323
for (const auto& part : subset.Flatten) {
2424
stats.IndexSize.Add(part->IndexesRawSize, part->Label.Channel());
25+
stats.ByKeyFilterSize += part->ByKey ? part->ByKey->Raw.size() : 0;
2526
TAutoPtr<TStatsScreenedPartIterator> iter = new TStatsScreenedPartIterator(part, env, subset.Scheme->Keys, part->Small, part->Large,
2627
rowCountResolution / resolutionDivider, dataSizeResolution / resolutionDivider);
2728
auto ready = iter->Start();

‎ydb/core/tx/datashard/datashard__stats.cpp

+14-14
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,6 @@ class TTableStatsCoroBuilder : public TActorCoroImpl, private IPages {
133133

134134
auto ev = MakeHolder<TDataShard::TEvPrivate::TEvAsyncTableStats>();
135135
ev->TableId = TableId;
136-
ev->IndexSize = IndexSize;
137136
ev->StatsUpdateTime = StatsUpdateTime;
138137
ev->PartCount = Subset->Flatten.size() + Subset->ColdParts.size();
139138
ev->MemRowCount = MemRowCount;
@@ -272,17 +271,14 @@ class TDataShard::TTxGetTableStats : public NTabletFlatExecutor::TTransactionBas
272271

273272
const TUserTable& tableInfo = *Self->TableInfos[tableId];
274273

275-
auto indexSize = txc.DB.GetTableIndexSize(tableInfo.LocalTid);
274+
// Fill stats with current mem table size:
276275
auto memSize = txc.DB.GetTableMemSize(tableInfo.LocalTid);
277276
auto memRowCount = txc.DB.GetTableMemRowCount(tableInfo.LocalTid);
278-
279277
if (tableInfo.ShadowTid) {
280-
indexSize += txc.DB.GetTableIndexSize(tableInfo.ShadowTid);
281278
memSize += txc.DB.GetTableMemSize(tableInfo.ShadowTid);
282279
memRowCount += txc.DB.GetTableMemRowCount(tableInfo.ShadowTid);
283280
}
284281

285-
Result->Record.MutableTableStats()->SetIndexSize(indexSize);
286282
Result->Record.MutableTableStats()->SetInMemSize(memSize);
287283
Result->Record.MutableTableStats()->SetLastAccessTime(tableInfo.Stats.AccessTime.MilliSeconds());
288284
Result->Record.MutableTableStats()->SetLastUpdateTime(tableInfo.Stats.UpdateTime.MilliSeconds());
@@ -291,18 +287,21 @@ class TDataShard::TTxGetTableStats : public NTabletFlatExecutor::TTransactionBas
291287
tableInfo.Stats.RowCountResolution = Ev->Get()->Record.GetRowCountResolution();
292288
tableInfo.Stats.HistogramBucketsCount = Ev->Get()->Record.GetHistogramBucketsCount();
293289

294-
// Check if first stats update has been completed
290+
// Check if first stats update has been completed:
295291
bool ready = (tableInfo.Stats.StatsUpdateTime != TInstant());
296292
Result->Record.SetFullStatsReady(ready);
297-
if (!ready)
293+
if (!ready) {
298294
return true;
295+
}
299296

300297
const TStats& stats = tableInfo.Stats.DataStats;
298+
Result->Record.MutableTableStats()->SetIndexSize(stats.IndexSize.Size);
299+
Result->Record.MutableTableStats()->SetByKeyFilterSize(stats.ByKeyFilterSize);
301300
Result->Record.MutableTableStats()->SetDataSize(stats.DataSize.Size + memSize);
302301
Result->Record.MutableTableStats()->SetRowCount(stats.RowCount + memRowCount);
303302
FillHistogram(stats.DataSizeHistogram, *Result->Record.MutableTableStats()->MutableDataSizeHistogram());
304303
FillHistogram(stats.RowCountHistogram, *Result->Record.MutableTableStats()->MutableRowCountHistogram());
305-
// Fill key access sample if it was collected not too long ago
304+
// Fill key access sample if it was collected not too long ago:
306305
if (Self->StopKeyAccessSamplingAt + TDuration::Seconds(30) >= AppData(ctx)->TimeProvider->Now()) {
307306
FillKeyAccessSample(tableInfo.Stats.AccessStats, *Result->Record.MutableTableStats()->MutableKeyAccessSample());
308307
}
@@ -317,7 +316,7 @@ class TDataShard::TTxGetTableStats : public NTabletFlatExecutor::TTransactionBas
317316
Result->Record.AddUserTablePartOwners(pi);
318317
}
319318

320-
for (const auto& pi : Self->SysTablesPartOnwers) {
319+
for (const auto& pi : Self->SysTablesPartOwners) {
321320
Result->Record.AddSysTablesPartOwners(pi);
322321
}
323322

@@ -375,9 +374,10 @@ void TDataShard::Handle(TEvPrivate::TEvAsyncTableStats::TPtr& ev, const TActorCo
375374
LOG_ERROR(ctx, NKikimrServices::TX_DATASHARD,
376375
"Unexpected async stats update at datashard %" PRIu64, TabletID());
377376
}
378-
tableInfo.Stats.Update(std::move(ev->Get()->Stats), ev->Get()->IndexSize,
379-
std::move(ev->Get()->PartOwners), ev->Get()->PartCount,
380-
ev->Get()->StatsUpdateTime);
377+
tableInfo.Stats.DataStats = std::move(ev->Get()->Stats);
378+
tableInfo.Stats.PartOwners = std::move(ev->Get()->PartOwners);
379+
tableInfo.Stats.PartCount = ev->Get()->PartCount;
380+
tableInfo.Stats.StatsUpdateTime = ev->Get()->StatsUpdateTime;
381381
tableInfo.Stats.MemRowCount = ev->Get()->MemRowCount;
382382
tableInfo.Stats.MemDataSize = ev->Get()->MemDataSize;
383383

@@ -565,12 +565,12 @@ class TDataShard::TTxInitiateStatsUpdate : public NTabletFlatExecutor::TTransact
565565
Self->Actors.insert(actorId);
566566
}
567567

568-
Self->SysTablesPartOnwers.clear();
568+
Self->SysTablesPartOwners.clear();
569569
for (ui32 sysTableId : Self->SysTablesToTransferAtSplit) {
570570
THashSet<ui64> sysPartOwners;
571571
auto subset = txc.DB.Subset(sysTableId, TEpoch::Max(), { }, { });
572572
GetPartOwners(*subset, sysPartOwners);
573-
Self->SysTablesPartOnwers.insert(sysPartOwners.begin(), sysPartOwners.end());
573+
Self->SysTablesPartOwners.insert(sysPartOwners.begin(), sysPartOwners.end());
574574
}
575575
return true;
576576
}

‎ydb/core/tx/datashard/datashard_impl.h

+4-4
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,6 @@ class TDataShard
403403

404404
struct TEvAsyncTableStats : public TEventLocal<TEvAsyncTableStats, EvAsyncTableStats> {
405405
ui64 TableId = -1;
406-
ui64 IndexSize = 0;
407406
TInstant StatsUpdateTime;
408407
NTable::TStats Stats;
409408
THashSet<ui64> PartOwners;
@@ -2634,7 +2633,7 @@ class TDataShard
26342633
Schema::PlanQueue::TableId,
26352634
Schema::DeadlineQueue::TableId
26362635
};
2637-
THashSet<ui64> SysTablesPartOnwers;
2636+
THashSet<ui64> SysTablesPartOwners;
26382637

26392638
// Sys table contents
26402639
ui32 State;
@@ -2649,7 +2648,7 @@ class TDataShard
26492648

26502649
NMiniKQL::IKeyAccessSampler::TPtr DisabledKeySampler;
26512650
NMiniKQL::IKeyAccessSampler::TPtr EnabledKeySampler;
2652-
NMiniKQL::IKeyAccessSampler::TPtr CurrentKeySampler; // Points to enbaled or disabled
2651+
NMiniKQL::IKeyAccessSampler::TPtr CurrentKeySampler; // Points to enabled or disabled
26532652
TInstant StartedKeyAccessSamplingAt;
26542653
TInstant StopKeyAccessSamplingAt;
26552654

@@ -3241,6 +3240,7 @@ class TDataShard
32413240

32423241
ev->Record.MutableTableStats()->SetDataSize(ti.Stats.DataStats.DataSize.Size + ti.Stats.MemDataSize);
32433242
ev->Record.MutableTableStats()->SetIndexSize(ti.Stats.DataStats.IndexSize.Size);
3243+
ev->Record.MutableTableStats()->SetByKeyFilterSize(ti.Stats.DataStats.ByKeyFilterSize);
32443244
ev->Record.MutableTableStats()->SetInMemSize(ti.Stats.MemDataSize);
32453245

32463246
TMap<ui8, std::tuple<ui64, ui64>> channels; // Channel -> (DataSize, IndexSize)
@@ -3294,7 +3294,7 @@ class TDataShard
32943294
for (const auto& pi : ti.Stats.PartOwners) {
32953295
ev->Record.AddUserTablePartOwners(pi);
32963296
}
3297-
for (const auto& pi : SysTablesPartOnwers) {
3297+
for (const auto& pi : SysTablesPartOwners) {
32983298
ev->Record.AddSysTablesPartOwners(pi);
32993299
}
33003300

‎ydb/core/tx/datashard/datashard_user_table.h

-9
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,6 @@ struct TUserTable : public TThrRefBase {
354354

355355
struct TStats {
356356
NTable::TStats DataStats;
357-
ui64 IndexSize = 0;
358357
ui64 MemRowCount = 0;
359358
ui64 MemDataSize = 0;
360359
TInstant AccessTime;
@@ -371,14 +370,6 @@ struct TUserTable : public TThrRefBase {
371370
ui64 BackgroundCompactionCount = 0;
372371
ui64 CompactBorrowedCount = 0;
373372
NTable::TKeyAccessSample AccessStats;
374-
375-
void Update(NTable::TStats&& dataStats, ui64 indexSize, THashSet<ui64>&& partOwners, ui64 partCount, TInstant statsUpdateTime) {
376-
DataStats = dataStats;
377-
IndexSize = indexSize;
378-
PartOwners = partOwners;
379-
PartCount = partCount;
380-
StatsUpdateTime = statsUpdateTime;
381-
}
382373
};
383374

384375
struct TSpecialUpdate {

‎ydb/core/tx/schemeshard/schemeshard__init.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -2285,6 +2285,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
22852285
stats.RowCount = rowSet.GetValue<Schema::TablePartitionStats::RowCount>();
22862286
stats.DataSize = rowSet.GetValue<Schema::TablePartitionStats::DataSize>();
22872287
stats.IndexSize = rowSet.GetValue<Schema::TablePartitionStats::IndexSize>();
2288+
stats.ByKeyFilterSize = rowSet.GetValue<Schema::TablePartitionStats::ByKeyFilterSize>();
22882289
if (rowSet.HaveValue<Schema::TablePartitionStats::StoragePoolsStats>()) {
22892290
NKikimrTableStats::TStoragePoolsStats protobufRepresentation;
22902291
Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(

‎ydb/core/tx/schemeshard/schemeshard__table_stats.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ auto TSchemeShard::BuildStatsForCollector(TPathId pathId, TShardIdx shardIdx, TT
6464
sysStats.SetDataSize(stats.DataSize);
6565
sysStats.SetRowCount(stats.RowCount);
6666
sysStats.SetIndexSize(stats.IndexSize);
67+
sysStats.SetByKeyFilterSize(stats.ByKeyFilterSize);
6768
sysStats.SetCPUCores(std::min(stats.GetCurrentRawCpuUsage() / 1000000., 1.0));
6869
sysStats.SetTabletId(ui64(datashardId));
6970
sysStats.SetAccessTime(stats.LastAccessTime.MilliSeconds());
@@ -164,6 +165,7 @@ TPartitionStats TTxStoreTableStats::PrepareStats(const TActorContext& ctx,
164165
newStats.RowCount = tableStats.GetRowCount();
165166
newStats.DataSize = tableStats.GetDataSize();
166167
newStats.IndexSize = tableStats.GetIndexSize();
168+
newStats.ByKeyFilterSize = tableStats.GetByKeyFilterSize();
167169
newStats.LastAccessTime = TInstant::MilliSeconds(tableStats.GetLastAccessTime());
168170
newStats.LastUpdateTime = TInstant::MilliSeconds(tableStats.GetLastUpdateTime());
169171
for (const auto& channelStats : tableStats.GetChannels()) {

‎ydb/core/tx/schemeshard/schemeshard_impl.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -2506,6 +2506,7 @@ void TSchemeShard::PersistTablePartitionStats(NIceDb::TNiceDb& db, const TPathId
25062506
NIceDb::TUpdate<Schema::TablePartitionStats::RowCount>(stats.RowCount),
25072507
NIceDb::TUpdate<Schema::TablePartitionStats::DataSize>(stats.DataSize),
25082508
NIceDb::TUpdate<Schema::TablePartitionStats::IndexSize>(stats.IndexSize),
2509+
NIceDb::TUpdate<Schema::TablePartitionStats::ByKeyFilterSize>(stats.ByKeyFilterSize),
25092510

25102511
NIceDb::TUpdate<Schema::TablePartitionStats::LastAccessTime>(stats.LastAccessTime.GetValue()),
25112512
NIceDb::TUpdate<Schema::TablePartitionStats::LastUpdateTime>(stats.LastUpdateTime.GetValue()),

‎ydb/core/tx/schemeshard/schemeshard_info_types.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -1592,6 +1592,7 @@ void TTableInfo::SetPartitioning(TVector<TTableShardInfo>&& newPartitioning) {
15921592
newAggregatedStats.RowCount += newStats.RowCount;
15931593
newAggregatedStats.DataSize += newStats.DataSize;
15941594
newAggregatedStats.IndexSize += newStats.IndexSize;
1595+
newAggregatedStats.ByKeyFilterSize += newStats.ByKeyFilterSize;
15951596
for (const auto& [poolKind, newStoragePoolStats] : newStats.StoragePoolsStats) {
15961597
auto& [dataSize, indexSize] = newAggregatedStats.StoragePoolsStats[poolKind];
15971598
dataSize += newStoragePoolStats.DataSize;
@@ -1678,6 +1679,7 @@ void TAggregatedStats::UpdateShardStats(TShardIdx datashardIdx, const TPartition
16781679
Aggregated.RowCount += (newStats.RowCount - oldStats.RowCount);
16791680
Aggregated.DataSize += (newStats.DataSize - oldStats.DataSize);
16801681
Aggregated.IndexSize += (newStats.IndexSize - oldStats.IndexSize);
1682+
Aggregated.ByKeyFilterSize += (newStats.ByKeyFilterSize - oldStats.ByKeyFilterSize);
16811683
for (const auto& [poolKind, newStoragePoolStats] : newStats.StoragePoolsStats) {
16821684
auto& [dataSize, indexSize] = Aggregated.StoragePoolsStats[poolKind];
16831685
const auto* oldStoragePoolStats = oldStats.StoragePoolsStats.FindPtr(poolKind);

‎ydb/core/tx/schemeshard/schemeshard_info_types.h

+1
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ struct TPartitionStats {
231231
ui64 RowCount = 0;
232232
ui64 DataSize = 0;
233233
ui64 IndexSize = 0;
234+
ui64 ByKeyFilterSize = 0;
234235

235236
struct TStoragePoolStats {
236237
ui64 DataSize = 0;

‎ydb/core/tx/schemeshard/schemeshard_path_describer.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ static void FillTableStats(NKikimrTableStats::TTableStats* stats, const TPartiti
2525
stats->SetRowCount(tableStats.RowCount);
2626
stats->SetDataSize(tableStats.DataSize);
2727
stats->SetIndexSize(tableStats.IndexSize);
28+
stats->SetByKeyFilterSize(tableStats.ByKeyFilterSize);
2829
stats->SetLastAccessTime(tableStats.LastAccessTime.MilliSeconds());
2930
stats->SetLastUpdateTime(tableStats.LastUpdateTime.MilliSeconds());
3031
stats->SetImmediateTxCompleted(tableStats.ImmediateTxCompleted);

‎ydb/core/tx/schemeshard/schemeshard_schema.h

+4-1
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,8 @@ struct Schema : NIceDb::Schema {
378378
// Represented by NKikimrTableStats::TStoragePoolsStats.
379379
struct StoragePoolsStats : Column<33, NScheme::NTypeIds::String> { using Type = TString; };
380380

381+
struct ByKeyFilterSize : Column<34, NScheme::NTypeIds::Uint64> {};
382+
381383
using TKey = TableKey<TableOwnerId, TableLocalId, PartitionId>;
382384
using TColumns = TableColumns<
383385
TableOwnerId,
@@ -412,7 +414,8 @@ struct Schema : NIceDb::Schema {
412414
SearchHeight,
413415
FullCompactionTs,
414416
MemDataSize,
415-
StoragePoolsStats
417+
StoragePoolsStats,
418+
ByKeyFilterSize
416419
>;
417420
};
418421

‎ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema

+7-1
Original file line numberDiff line numberDiff line change
@@ -6828,6 +6828,11 @@
68286828
"ColumnId": 33,
68296829
"ColumnName": "StoragePoolsStats",
68306830
"ColumnType": "String"
6831+
},
6832+
{
6833+
"ColumnId": 34,
6834+
"ColumnName": "ByKeyFilterSize",
6835+
"ColumnType": "Uint64"
68316836
}
68326837
],
68336838
"ColumnsDropped": [],
@@ -6866,7 +6871,8 @@
68666871
30,
68676872
31,
68686873
32,
6869-
33
6874+
33,
6875+
34
68706876
],
68716877
"RoomID": 0,
68726878
"Codec": 0,

0 commit comments

Comments
 (0)