Skip to content

Commit 897ada7

Browse files
kungakunga
committed
KIKIMR-19521 BTreeIndex Loader & Dump
1 parent 33afeca commit 897ada7

8 files changed

+68
-31
lines changed

‎ydb/core/tablet_flat/flat_part_btree_index_iter.h

+8
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,14 @@ class TPartBtreeIndexIt : public IIndexIter {
129129
return DoSeek<TSeekRowId>({rowId});
130130
}
131131

132+
EReady SeekLast() override {
133+
if (Y_UNLIKELY(GetEndRowId() == 0)) {
134+
Y_DEBUG_ABORT_UNLESS(false, "TPart can't be empty");
135+
return Exhaust();
136+
}
137+
return Seek(GetEndRowId() - 1);
138+
}
139+
132140
/**
133141
* Searches for the first page that may contain given key with specified seek mode
134142
*

‎ydb/core/tablet_flat/flat_part_dump.cpp

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#include "flat_part_dump.h"
22
#include "flat_part_iface.h"
3-
#include "flat_part_index_iter.h"
3+
#include "flat_part_index_iter_iface.h"
44
#include "flat_page_data.h"
55
#include "flat_page_frames.h"
66
#include "flat_page_blobs.h"
@@ -52,10 +52,10 @@ namespace {
5252
BTreeIndex(part);
5353

5454
if (depth > 2) {
55-
auto index = TPartIndexIt(&part, Env, { });
55+
auto index = CreateIndexIter(&part, Env, { });
5656

5757
for (ssize_t i = 0; ; i++) {
58-
auto ready = i == 0 ? index.Seek(0) : index.Next();
58+
auto ready = i == 0 ? index->Seek(0) : index->Next();
5959
if (ready != EReady::Data) {
6060
if (ready == EReady::Page) {
6161
Out << " | -- the rest of the index rows aren't loaded" << Endl;
@@ -65,7 +65,7 @@ namespace {
6565

6666
Out << Endl;
6767

68-
DataPage(part, index.GetPageId());
68+
DataPage(part, index->GetPageId());
6969
}
7070
}
7171
}

‎ydb/core/tablet_flat/flat_part_index_iter.h

+11-11
Original file line numberDiff line numberDiff line change
@@ -36,36 +36,36 @@ class TPartIndexIt : public IIndexIter {
3636
return DataOrGone();
3737
}
3838

39-
EReady Seek(ESeek seek, TCells key, const TKeyCellDefaults *keyDefaults) override {
39+
EReady SeekLast() override {
4040
auto index = TryGetIndex();
4141
if (!index) {
4242
return EReady::Page;
4343
}
44-
45-
Iter = index->LookupKey(key, GroupInfo, seek, keyDefaults);
44+
Iter = (*index)->End();
45+
if (Iter.Off() == 0) {
46+
return EReady::Gone;
47+
}
48+
Iter--;
4649
return DataOrGone();
4750
}
4851

49-
EReady SeekReverse(ESeek seek, TCells key, const TKeyCellDefaults *keyDefaults) override {
52+
EReady Seek(ESeek seek, TCells key, const TKeyCellDefaults *keyDefaults) override {
5053
auto index = TryGetIndex();
5154
if (!index) {
5255
return EReady::Page;
5356
}
5457

55-
Iter = index->LookupKeyReverse(key, GroupInfo, seek, keyDefaults);
58+
Iter = index->LookupKey(key, GroupInfo, seek, keyDefaults);
5659
return DataOrGone();
5760
}
5861

59-
EReady SeekLast() {
62+
EReady SeekReverse(ESeek seek, TCells key, const TKeyCellDefaults *keyDefaults) override {
6063
auto index = TryGetIndex();
6164
if (!index) {
6265
return EReady::Page;
6366
}
64-
Iter = (*index)->End();
65-
if (Iter.Off() == 0) {
66-
return EReady::Gone;
67-
}
68-
Iter--;
67+
68+
Iter = index->LookupKeyReverse(key, GroupInfo, seek, keyDefaults);
6969
return DataOrGone();
7070
}
7171

‎ydb/core/tablet_flat/flat_part_index_iter_iface.h

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ namespace NKikimr::NTable {
99
using TCells = NPage::TCells;
1010

1111
virtual EReady Seek(TRowId rowId) = 0;
12+
virtual EReady SeekLast() = 0;
1213
virtual EReady Seek(ESeek seek, TCells key, const TKeyCellDefaults *keyDefaults) = 0;
1314
virtual EReady SeekReverse(ESeek seek, TCells key, const TKeyCellDefaults *keyDefaults) = 0;
1415
virtual EReady Next() = 0;

‎ydb/core/tablet_flat/flat_part_keys.h

+10-11
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
#pragma once
22
#include "flat_part_iface.h"
3-
#include "flat_part_index_iter.h"
3+
#include "flat_part_index_iter_iface.h"
44
#include "flat_part_slice.h"
55
#include "flat_sausage_fetch.h"
66
#include "flat_sausagecache.h"
7-
#include "util_fmt_abort.h"
87

98
namespace NKikimr {
109
namespace NTable {
@@ -82,7 +81,7 @@ namespace NTable {
8281
explicit TKeysLoader(const TPart* part, IPages* env)
8382
: Part(part)
8483
, Env(env)
85-
, Index(Part, Env, {})
84+
, Index(CreateIndexIter(part, env, {}))
8685
{
8786
}
8887

@@ -151,7 +150,7 @@ namespace NTable {
151150
return true;
152151
}
153152

154-
auto ready = Index.Seek(rowId);
153+
auto ready = Index->Seek(rowId);
155154
if (ready == EReady::Page) {
156155
return false;
157156
} else if (ready == EReady::Gone) {
@@ -161,11 +160,11 @@ namespace NTable {
161160
return true;
162161
}
163162

164-
Y_ABORT_UNLESS(Index.GetRowId() <= rowId, "SeekIndex invariant failure");
165-
if (!LoadPage(Index.GetPageId())) {
163+
Y_ABORT_UNLESS(Index->GetRowId() <= rowId, "SeekIndex invariant failure");
164+
if (!LoadPage(Index->GetPageId())) {
166165
return false;
167166
}
168-
Y_ABORT_UNLESS(Page.BaseRow() == Index.GetRowId(), "Index and data are out of sync");
167+
Y_ABORT_UNLESS(Page.BaseRow() == Index->GetRowId(), "Index and data are out of sync");
169168
auto lastRowId = Page.BaseRow() + (Page->Count - 1);
170169
if (lastRowId < rowId) {
171170
// Row is out of range for this page
@@ -180,16 +179,16 @@ namespace NTable {
180179

181180
bool SeekLastRow() noexcept
182181
{
183-
auto hasLast = Index.SeekLast();
182+
auto hasLast = Index->SeekLast();
184183
if (hasLast == EReady::Page) {
185184
return false;
186185
}
187186
Y_ABORT_UNLESS(hasLast != EReady::Gone, "Unexpected failure to find the last index record");
188187

189-
if (!LoadPage(Index.GetPageId())) {
188+
if (!LoadPage(Index->GetPageId())) {
190189
return false;
191190
}
192-
Y_ABORT_UNLESS(Page.BaseRow() == Index.GetRowId(), "Index and data are out of sync");
191+
Y_ABORT_UNLESS(Page.BaseRow() == Index->GetRowId(), "Index and data are out of sync");
193192
auto lastRowId = Page.BaseRow() + (Page->Count - 1);
194193
LoadRow(lastRowId);
195194
return true;
@@ -227,7 +226,7 @@ namespace NTable {
227226
IPages* Env;
228227
TRowId RowId = Max<TRowId>();
229228
TPageId PageId = Max<TPageId>();
230-
TPartIndexIt Index;
229+
THolder<IIndexIter> Index;
231230
NPage::TDataPage Page;
232231
TSmallVec<TCell> Key;
233232
};

‎ydb/core/tablet_flat/ut/ut_btree_index.cpp

+22
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,13 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) {
915915
}, message, failsAllowed);
916916
}
917917

918+
template<typename TIter>
919+
EReady SeekLast(TIter& iter, const TString& message, ui32 failsAllowed = 10) {
920+
return Retry([&]() {
921+
return iter.SeekLast();
922+
}, message, failsAllowed);
923+
}
924+
918925
template<typename TIter>
919926
EReady SeekKey(TIter& iter, ESeek seek, bool reverse, TCells key, const TKeyCellDefaults *keyDefaults, const TString& message, ui32 failsAllowed = 10) {
920927
return Retry([&]() {
@@ -966,6 +973,19 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) {
966973
}
967974
}
968975

976+
template<typename TEnv>
977+
void CheckSeekLast(const TPartStore& part) {
978+
TEnv env;
979+
TPartBtreeIndexIt bTree(&part, &env, { });
980+
TPartIndexIt flat(&part, &env, { });
981+
982+
TString message = TStringBuilder() << "SeekLast<" << typeid(TEnv).name() << ">";
983+
EReady bTreeReady = SeekLast(bTree, message);
984+
EReady flatReady = SeekLast(flat, message);
985+
UNIT_ASSERT_VALUES_EQUAL(bTreeReady, EReady::Data);
986+
AssertEqual(bTree, bTreeReady, flat, flatReady, message);
987+
}
988+
969989
template<typename TEnv>
970990
void CheckSeekKey(const TPartStore& part, const TKeyCellDefaults *keyDefaults) {
971991
for (bool reverse : {false, true}) {
@@ -1055,6 +1075,8 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) {
10551075

10561076
CheckSeekRowId<TTestEnv>(part);
10571077
CheckSeekRowId<TTouchEnv>(part);
1078+
CheckSeekLast<TTestEnv>(part);
1079+
CheckSeekLast<TTouchEnv>(part);
10581080
CheckSeekKey<TTestEnv>(part, eggs.Scheme->Keys.Get());
10591081
CheckSeekKey<TTouchEnv>(part, eggs.Scheme->Keys.Get());
10601082
CheckNextPrev<TTestEnv>(part);

‎ydb/core/tablet_flat/ut/ut_slice_loader.cpp

+11-5
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ namespace {
2626
NPage::TConf conf{ true, 2 * 1024 };
2727

2828
conf.Group(0).IndexMin = 1024; /* Should cover index buffer grow code */
29+
conf.Group(0).BTreeIndexNodeTargetSize = 512; /* Should cover up/down moves */
2930
conf.SmallEdge = 19; /* Packed to page collection large cell values */
3031
conf.LargeEdge = 29; /* Large values placed to single blobs */
3132
conf.SliceSize = conf.Group(0).PageSize * 4;
@@ -53,6 +54,11 @@ namespace {
5354
return part;
5455
}
5556

57+
bool IsBTreeIndex()
58+
{
59+
return Eggs0().Lone()->IndexPages.BTreeGroups.size();
60+
}
61+
5662
class TTestPartPageCollection : public NPageCollection::IPageCollection {
5763
public:
5864
TTestPartPageCollection(TIntrusiveConstPtr<NTest::TPartStore> part, ui32 room)
@@ -187,7 +193,7 @@ Y_UNIT_TEST_SUITE(TPartSliceLoader) {
187193

188194
Y_UNIT_TEST(RestoreMissingSlice) {
189195
auto result = RunLoaderTest(Part0(), nullptr);
190-
UNIT_ASSERT_C(result.Pages == 3, // index + first + last
196+
UNIT_ASSERT_C(result.Pages == (IsBTreeIndex() ? 5 : 1) + 2, // index + first + last
191197
"Restoring slice bounds needed " << result.Pages << " extra pages");
192198
}
193199

@@ -199,7 +205,7 @@ Y_UNIT_TEST_SUITE(TPartSliceLoader) {
199205
TIntrusiveConstPtr<TScreen> screen = new TScreen(std::move(holes));
200206
auto result = RunLoaderTest(Part0(), screen);
201207

202-
UNIT_ASSERT_VALUES_EQUAL_C(result.Pages, 3, // index + first + last
208+
UNIT_ASSERT_VALUES_EQUAL_C(result.Pages, (IsBTreeIndex() ? 5 : 1) + 2, // index + first + last
203209
"Restoring slice [" << startOff << ", " << IndexTools::GetEndRowId(*Part0()) + endOff << "] bounds needed "
204210
<< result.Pages << " extra pages");
205211
}
@@ -227,7 +233,7 @@ Y_UNIT_TEST_SUITE(TPartSliceLoader) {
227233
screen = new TScreen(std::move(holes));
228234
}
229235
auto result = RunLoaderTest(Part0(), screen);
230-
UNIT_ASSERT_VALUES_EQUAL_C(result.Pages, 1 + IndexTools::CountMainPages(*Part0()), // index + all data pages
236+
UNIT_ASSERT_VALUES_EQUAL_C(result.Pages, (IsBTreeIndex() ? 70 : 1) + IndexTools::CountMainPages(*Part0()), // index + all data pages
231237
"Restoring slice bounds needed " << result.Pages << " extra pages");
232238
}
233239

@@ -254,7 +260,7 @@ Y_UNIT_TEST_SUITE(TPartSliceLoader) {
254260
screen = new TScreen(std::move(holes));
255261
}
256262
auto result = RunLoaderTest(Part0(), screen);
257-
UNIT_ASSERT_VALUES_EQUAL_C(result.Pages, 1 + IndexTools::CountMainPages(*Part0()), // index + all data pages
263+
UNIT_ASSERT_VALUES_EQUAL_C(result.Pages, (IsBTreeIndex() ? 70 : 1) + IndexTools::CountMainPages(*Part0()), // index + all data pages
258264
"Restoring slice bounds needed " << result.Pages << " extra pages");
259265
}
260266

@@ -283,7 +289,7 @@ Y_UNIT_TEST_SUITE(TPartSliceLoader) {
283289
screen = new TScreen(std::move(holes));
284290
}
285291
auto result = RunLoaderTest(Part0(), screen);
286-
UNIT_ASSERT_VALUES_EQUAL_C(result.Pages, screen->Size() + 1, // index + data pages
292+
UNIT_ASSERT_VALUES_EQUAL_C(result.Pages, (IsBTreeIndex() ? 70 : 1) + screen->Size(), // index + data pages
287293
"Restoring slice bounds needed " << result.Pages <<
288294
" extra pages, expected " << screen->Size());
289295
}

‎ydb/core/tablet_flat/ut/ut_stat.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ namespace {
3030
conf.Groups.resize(groups);
3131
for (size_t group : xrange(groups)) {
3232
conf.Group(group).IndexMin = 1024; /* Should cover index buffer grow code */
33+
conf.Group(group).BTreeIndexNodeTargetSize = 512; /* Should cover up/down moves */
3334
}
3435
conf.SmallEdge = 19; /* Packed to page collection large cell values */
3536
conf.LargeEdge = 29; /* Large values placed to single blobs */

0 commit comments

Comments
 (0)