Skip to content

Commit 228b085

Browse files
kungakunga
authored
KIKIMR-19521 BTreeIndex Test (#653)
1 parent 8fbea85 commit 228b085

File tree

6 files changed

+381
-42
lines changed

6 files changed

+381
-42
lines changed

‎ydb/core/tablet_flat/benchmark/b_charge.cpp

Lines changed: 14 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -32,35 +32,10 @@ namespace {
3232
ui64 TouchedCount = 0;
3333
};
3434

35-
struct TCooker {
36-
TCooker(const TRowScheme &scheme)
37-
: Tool(scheme)
38-
, Writer(new TPartScheme(scheme.Cols), { }, NPage::TGroupId(0))
39-
{
40-
41-
}
42-
43-
TCooker& Add(const NTest::TRow &row, ui64 offset, ui32 page)
44-
{
45-
const TCelled key(Tool.LookupKey(row), *Tool.Scheme.Keys, false);
46-
47-
return Writer.Add(key, offset, page), *this;
48-
}
49-
50-
TSharedData Flush()
51-
{
52-
return Writer.Flush();
53-
}
54-
55-
private:
56-
const NTest::TRowTool Tool;
57-
NPage::TIndexWriter Writer;
58-
};
59-
60-
struct TModel : public benchmark::Fixture {
35+
struct TPrechargeFixture : public benchmark::Fixture {
6136
using TGroupId = NPage::TGroupId;
6237

63-
TModel()
38+
TPrechargeFixture()
6439
: Tool(*Mass.Model->Scheme)
6540
{
6641
Y_ABORT_UNLESS(NTest::IndexTools::CountMainPages(*Eggs.Lone()) > 120);
@@ -125,7 +100,7 @@ namespace {
125100
};
126101
}
127102

128-
BENCHMARK_DEFINE_F(TModel, PrechargeByKeys)(benchmark::State& state) {
103+
BENCHMARK_DEFINE_F(TPrechargeFixture, PrechargeByKeys)(benchmark::State& state) {
129104
ui64 items = state.range(0);
130105

131106
const auto &keyDefaults = *Tool.Scheme.Keys;
@@ -144,7 +119,7 @@ BENCHMARK_DEFINE_F(TModel, PrechargeByKeys)(benchmark::State& state) {
144119
state.counters["Touched"] = benchmark::Counter(Env->TouchedCount, benchmark::Counter::kAvgIterations);
145120
}
146121

147-
BENCHMARK_DEFINE_F(TModel, PrechargeByRows)(benchmark::State& state) {
122+
BENCHMARK_DEFINE_F(TPrechargeFixture, PrechargeByRows)(benchmark::State& state) {
148123
ui64 items = state.range(0);
149124

150125
const auto &keyDefaults = *Tool.Scheme.Keys;
@@ -160,12 +135,18 @@ BENCHMARK_DEFINE_F(TModel, PrechargeByRows)(benchmark::State& state) {
160135
state.counters["Touched"] = Env->TouchedCount / it;
161136
}
162137

163-
BENCHMARK_REGISTER_F(TModel, PrechargeByKeys)
164-
->ArgsProduct({/*items:*/ {0, 100, 1000}, /*fail:*/{0, 1}, /*groups:*/ {0, 1, 2}})
138+
BENCHMARK_REGISTER_F(TPrechargeFixture, PrechargeByKeys)
139+
->ArgsProduct({
140+
/* items: */ {0, 100, 1000},
141+
/* fail: */ {0, 1},
142+
/* groups: */ {0, 1, 2}})
165143
->Unit(benchmark::kMicrosecond);
166144

167-
BENCHMARK_REGISTER_F(TModel, PrechargeByRows)
168-
->ArgsProduct({/*items:*/ {0, 100, 1000}, /*fail:*/{0, 1}, /*groups:*/ {0, 1, 2}})
145+
BENCHMARK_REGISTER_F(TPrechargeFixture, PrechargeByRows)
146+
->ArgsProduct({
147+
/* items: */ {0, 100, 1000},
148+
/* fail: */{0, 1},
149+
/* groups: */ {0, 1, 2}})
169150
->Unit(benchmark::kMicrosecond);
170151

171152
}

‎ydb/core/tablet_flat/benchmark/b_part_index.cpp

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
#include <benchmark/benchmark.h>
2+
3+
#include <ydb/core/tablet_flat/flat_row_celled.h>
4+
#include <ydb/core/tablet_flat/flat_part_charge_range.h>
5+
#include <ydb/core/tablet_flat/flat_part_charge_create.h>
6+
#include <ydb/core/tablet_flat/test/libs/rows/cook.h>
7+
#include <ydb/core/tablet_flat/test/libs/rows/tool.h>
8+
#include <ydb/core/tablet_flat/test/libs/table/model/large.h>
9+
#include <ydb/core/tablet_flat/test/libs/table/test_make.h>
10+
#include <ydb/core/tablet_flat/test/libs/table/test_mixer.h>
11+
#include "ydb/core/tablet_flat/flat_part_btree_index_iter.h"
12+
#include "ydb/core/tablet_flat/test/libs/table/wrap_iter.h"
13+
#include <ydb/core/tablet_flat/test/libs/table/test_writer.h>
14+
#include <ydb/core/tablet_flat/test/libs/table/test_envs.h>
15+
#include <ydb/core/tablet_flat/test/libs/table/wrap_part.h>
16+
#include <ydb/core/tablet_flat/test/libs/table/test_steps.h>
17+
18+
namespace NKikimr {
19+
namespace NTable {
20+
21+
namespace {
22+
using namespace NTest;
23+
24+
using TCheckIt = TChecker<TWrapIter, TSubset>;
25+
using TCheckReverseIt = TChecker<TWrapReverseIter, TSubset>;
26+
27+
NPage::TConf PageConf(size_t groups, bool writeBTreeIndex) noexcept
28+
{
29+
NPage::TConf conf{ true, 1024 };
30+
31+
conf.Groups.resize(groups);
32+
for (size_t group : xrange(groups)) {
33+
conf.Group(group).PageSize = 1024;
34+
conf.Group(group).BTreeIndexNodeTargetSize = 1024;
35+
}
36+
37+
conf.WriteBTreeIndex = writeBTreeIndex;
38+
39+
conf.SliceSize = conf.Group(0).PageSize * 4;
40+
41+
return conf;
42+
}
43+
44+
struct TPartIndexSeekFixture : public benchmark::Fixture {
45+
using TGroupId = NPage::TGroupId;
46+
47+
void SetUp(const ::benchmark::State& state)
48+
{
49+
const bool groups = state.range(1);
50+
51+
TLayoutCook lay;
52+
53+
lay
54+
.Col(0, 0, NScheme::NTypeIds::Uint32)
55+
.Col(0, 1, NScheme::NTypeIds::Uint32)
56+
.Col(0, 2, NScheme::NTypeIds::Uint32)
57+
.Col(0, 3, NScheme::NTypeIds::Uint32)
58+
.Col(groups ? 1 : 0, 4, NScheme::NTypeIds::Uint32)
59+
.Key({0, 1, 2});
60+
61+
TPartCook cook(lay, PageConf(groups ? 2 : 1, true));
62+
63+
for (ui32 i = 0; (groups ? cook.GetDataBytes(0) + cook.GetDataBytes(1) : cook.GetDataBytes(0)) < 100ull*1024*1024; i++) {
64+
cook.Add(*TSchemedCookRow(*lay).Col(i / 10000, i / 100 % 100, i % 100, i, i));
65+
}
66+
67+
Eggs = cook.Finish();
68+
69+
const auto part = Eggs.Lone();
70+
71+
Cerr << "DataBytes = " << part->Stat.Bytes << " DataPages = " << IndexTools::CountMainPages(*part) << Endl;
72+
Cerr << "FlatIndexBytes = " << part->GetPageSize(part->IndexPages.Groups[groups ? 1 : 0], {}) << " BTreeIndexBytes = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].IndexSize << Endl;
73+
Cerr << "Levels = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].LevelsCount << Endl;
74+
75+
// 100 MB
76+
UNIT_ASSERT_GE(part->Stat.Bytes, 100ull*1024*1024);
77+
UNIT_ASSERT_LE(part->Stat.Bytes, 100ull*1024*1024 + 10ull*1024*1024);
78+
79+
GroupId = TGroupId(groups ? 1 : 0);
80+
}
81+
82+
TPartEggs Eggs;
83+
TTestEnv Env;
84+
TGroupId GroupId;
85+
};
86+
87+
struct TPartIndexIteratorFixture : public benchmark::Fixture {
88+
using TGroupId = NPage::TGroupId;
89+
90+
void SetUp(const ::benchmark::State& state)
91+
{
92+
const bool useBTree = state.range(0);
93+
const bool groups = state.range(1);
94+
const bool history = state.range(2);
95+
96+
Mass = new NTest::TMass(new NTest::TModelStd(groups), history ? 1000000 : 300000);
97+
Subset = TMake(*Mass, PageConf(Mass->Model->Scheme->Families.size(), useBTree)).Mixed(0, 1, TMixerOne{ }, history ? 0.7 : 0);
98+
99+
if (history) {
100+
Checker = new TCheckIt(*Subset, {new TTestEnv()}, TRowVersion(0, 8));
101+
CheckerReverse = new TCheckReverseIt(*Subset, {new TTestEnv()}, TRowVersion(0, 8));
102+
} else {
103+
Checker = new TCheckIt(*Subset, {new TTestEnv()});
104+
CheckerReverse = new TCheckReverseIt(*Subset, {new TTestEnv()});
105+
}
106+
}
107+
108+
TMersenne<ui64> Rnd;
109+
TAutoPtr<NTest::TMass> Mass;
110+
TAutoPtr<TSubset> Subset;
111+
TAutoPtr<TCheckIt> Checker;
112+
TAutoPtr<TCheckReverseIt> CheckerReverse;
113+
};
114+
}
115+
116+
BENCHMARK_DEFINE_F(TPartIndexSeekFixture, SeekRowId)(benchmark::State& state) {
117+
const bool useBTree = state.range(0);
118+
119+
for (auto _ : state) {
120+
THolder<IIndexIter> iter;
121+
122+
if (useBTree) {
123+
iter = MakeHolder<TPartBtreeIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
124+
} else {
125+
iter = MakeHolder<TPartIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
126+
}
127+
128+
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
129+
}
130+
}
131+
132+
BENCHMARK_DEFINE_F(TPartIndexSeekFixture, Next)(benchmark::State& state) {
133+
const bool useBTree = state.range(0);
134+
135+
THolder<IIndexIter> iter;
136+
137+
if (useBTree) {
138+
iter = MakeHolder<TPartBtreeIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
139+
} else {
140+
iter = MakeHolder<TPartIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
141+
}
142+
143+
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
144+
145+
for (auto _ : state) {
146+
if (!iter->IsValid()) {
147+
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
148+
}
149+
iter->Next();
150+
}
151+
}
152+
153+
BENCHMARK_DEFINE_F(TPartIndexSeekFixture, Prev)(benchmark::State& state) {
154+
const bool useBTree = state.range(0);
155+
156+
THolder<IIndexIter> iter;
157+
158+
if (useBTree) {
159+
iter = MakeHolder<TPartBtreeIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
160+
} else {
161+
iter = MakeHolder<TPartIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
162+
}
163+
164+
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
165+
166+
for (auto _ : state) {
167+
if (!iter->IsValid()) {
168+
iter->Seek(RandomNumber<ui32>(Eggs.Lone()->Stat.Rows));
169+
}
170+
iter->Prev();
171+
}
172+
}
173+
174+
BENCHMARK_DEFINE_F(TPartIndexSeekFixture, SeekKey)(benchmark::State& state) {
175+
const bool useBTree = state.range(0);
176+
const ESeek seek = ESeek(state.range(2));
177+
178+
for (auto _ : state) {
179+
THolder<IIndexIter> iter;
180+
181+
if (useBTree) {
182+
iter = MakeHolder<TPartBtreeIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
183+
} else {
184+
iter = MakeHolder<TPartIndexIt>(Eggs.Lone().Get(), &Env, GroupId);
185+
}
186+
187+
ui32 rowId = RandomNumber<ui32>(Eggs.Lone()->Stat.Rows);
188+
TVector<TCell> key{TCell::Make(rowId / 10000), TCell::Make(rowId / 100 % 100), TCell::Make(rowId % 100)};
189+
iter->Seek(seek, key, Eggs.Scheme->Keys.Get());
190+
}
191+
}
192+
193+
BENCHMARK_DEFINE_F(TPartIndexIteratorFixture, DoReads)(benchmark::State& state) {
194+
const bool reverse = state.range(3);
195+
const ui32 items = state.range(4);
196+
197+
for (auto _ : state) {
198+
auto it = Mass->Saved.Any(Rnd);
199+
200+
if (reverse) {
201+
CheckerReverse->Seek(*it, ESeek::Lower);
202+
for (ui32 i = 1; CheckerReverse->GetReady() == EReady::Data && i < items; i++) {
203+
CheckerReverse->Next();
204+
}
205+
} else {
206+
Checker->Seek(*it, ESeek::Lower);
207+
for (ui32 i = 1; Checker->GetReady() == EReady::Data && i < items; i++) {
208+
Checker->Next();
209+
}
210+
}
211+
}
212+
}
213+
214+
BENCHMARK_REGISTER_F(TPartIndexSeekFixture, SeekRowId)
215+
->ArgsProduct({
216+
/* b-tree */ {0, 1},
217+
/* groups: */ {0, 1}})
218+
->Unit(benchmark::kMicrosecond);
219+
220+
BENCHMARK_REGISTER_F(TPartIndexSeekFixture, Next)
221+
->ArgsProduct({
222+
/* b-tree */ {0, 1},
223+
/* groups: */ {0, 1}})
224+
->Unit(benchmark::kMicrosecond);
225+
226+
BENCHMARK_REGISTER_F(TPartIndexSeekFixture, Prev)
227+
->ArgsProduct({
228+
/* b-tree */ {0, 1},
229+
/* groups: */ {0, 1}})
230+
->Unit(benchmark::kMicrosecond);
231+
232+
BENCHMARK_REGISTER_F(TPartIndexSeekFixture, SeekKey)
233+
->ArgsProduct({
234+
/* b-tree */ {0, 1},
235+
/* groups: */ {0, 1},
236+
/* ESeek: */ {0, 1, 2}})
237+
->Unit(benchmark::kMicrosecond);
238+
239+
BENCHMARK_REGISTER_F(TPartIndexIteratorFixture, DoReads)
240+
->ArgsProduct({
241+
/* b-tree */ {0, 1},
242+
/* groups: */ {0, 1},
243+
/* history: */ {0, 1},
244+
/* reverse: */ {0, 1},
245+
/* items */ {1, 10, 100}})
246+
->Unit(benchmark::kMicrosecond);
247+
248+
}
249+
}

‎ydb/core/tablet_flat/benchmark/ya.make

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
G_BENCHMARK()
22

3-
SIZE(MEDIUM)
4-
TIMEOUT(600)
3+
TAG(ya:fat)
4+
SIZE(LARGE)
5+
TIMEOUT(1200)
56

67
SRCS(
78
b_charge.cpp
9+
b_part_index.cpp
810
)
911

1012
PEERDIR(

‎ydb/core/tablet_flat/flat_part_btree_index_iter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ class TPartBtreeIndexIt : public IIndexIter {
116116
, GroupId(groupId)
117117
, GroupInfo(part->Scheme->GetLayout(groupId))
118118
, Meta(groupId.IsHistoric() ? part->IndexPages.BTreeHistoric[groupId.Index] : part->IndexPages.BTreeGroups[groupId.Index])
119+
, State(Reserve(Meta.LevelsCount + 1))
119120
{
120121
const static TCellsIterable EmptyKey(static_cast<const char*>(nullptr), TColumns());
121122
State.emplace_back(Meta, 0, GetEndRowId(), EmptyKey, EmptyKey);

‎ydb/core/tablet_flat/flat_part_loader.cpp

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,15 @@ TAutoPtr<NPageCollection::TFetch> TLoader::StageCreatePartView() noexcept
184184
// TODO: put index size to stat?
185185
// TODO: include history indexes bytes
186186
size_t indexesRawSize = 0;
187-
for (auto indexPage : groupIndexesIds) {
188-
indexesRawSize += GetPageSize(indexPage);
187+
if (AppData()->FeatureFlags.GetEnableLocalDBBtreeIndex()) {
188+
for (const auto &meta : BTreeGroupIndexes) {
189+
indexesRawSize += meta.IndexSize;
190+
}
191+
// Note: although we also have flat index, it shouldn't be loaded; so let's not count it here
192+
} else {
193+
for (auto indexPage : groupIndexesIds) {
194+
indexesRawSize += GetPageSize(indexPage);
195+
}
189196
}
190197

191198
auto *partStore = new TPartStore(

0 commit comments

Comments
 (0)