From 4e59b12a37036e25dcf328bf412255b390a299c3 Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 26 Dec 2023 09:12:58 +0000 Subject: [PATCH 01/11] int32 levels count --- ydb/core/tablet_flat/flat_page_btree_index.h | 2 +- ydb/core/tablet_flat/flat_page_btree_index_writer.h | 5 +++-- ydb/core/tablet_flat/flat_part_btree_index_iter.h | 7 +++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ydb/core/tablet_flat/flat_page_btree_index.h b/ydb/core/tablet_flat/flat_page_btree_index.h index 968fb5f9c742..794a517a5ad1 100644 --- a/ydb/core/tablet_flat/flat_page_btree_index.h +++ b/ydb/core/tablet_flat/flat_page_btree_index.h @@ -464,7 +464,7 @@ namespace NKikimr::NTable::NPage { }; struct TBtreeIndexMeta : public TBtreeIndexNode::TChild { - size_t LevelsCount; + ui32 LevelsCount; ui64 IndexSize; auto operator<=>(const TBtreeIndexMeta&) const = default; diff --git a/ydb/core/tablet_flat/flat_page_btree_index_writer.h b/ydb/core/tablet_flat/flat_page_btree_index_writer.h index 0c7b084f0556..bef4b93ceb71 100644 --- a/ydb/core/tablet_flat/flat_page_btree_index_writer.h +++ b/ydb/core/tablet_flat/flat_page_btree_index_writer.h @@ -388,7 +388,8 @@ namespace NKikimr::NTable::NPage { } std::optional Flush(IPageWriter &pager, bool last) { - for (size_t levelIndex = 0; levelIndex < Levels.size(); levelIndex++) { + Y_ABORT_UNLESS(Levels.size() < Max(), "Levels size is out of bounds"); + for (ui32 levelIndex = 0; levelIndex < Levels.size(); levelIndex++) { if (last && !Levels[levelIndex].GetKeysCount()) { Y_ABORT_UNLESS(Levels[levelIndex].GetChildrenCount() == 1, "Should be root"); return TBtreeIndexMeta{ Levels[levelIndex].PopChild(), levelIndex, IndexSize }; @@ -414,7 +415,7 @@ namespace NKikimr::NTable::NPage { } private: - bool TryFlush(size_t levelIndex, IPageWriter &pager, bool last) { + bool TryFlush(ui32 levelIndex, IPageWriter &pager, bool last) { if (!last && Levels[levelIndex].GetKeysCount() <= 2 * NodeKeysMax) { // Note: node should meet both NodeKeysMin and NodeSize restrictions for split diff --git a/ydb/core/tablet_flat/flat_part_btree_index_iter.h b/ydb/core/tablet_flat/flat_part_btree_index_iter.h index 92c5c69f2cb0..df10bc376426 100644 --- a/ydb/core/tablet_flat/flat_part_btree_index_iter.h +++ b/ydb/core/tablet_flat/flat_part_btree_index_iter.h @@ -1,7 +1,6 @@ #pragma once #include "flat_part_iface.h" -#include "flat_page_index.h" #include "flat_table_part.h" #include "flat_part_index_iter_iface.h" @@ -195,7 +194,7 @@ class TPartBtreeIndexIt : public IIndexIter { PushNextState(*State.back().Pos + 1); } - for (size_t level : xrange(State.size() - 1, Meta.LevelsCount)) { + for (ui32 level : xrange(State.size() - 1, Meta.LevelsCount)) { if (!TryLoad(State[level])) { // exiting with an intermediate state Y_DEBUG_ABORT_UNLESS(!IsLeaf() && !IsExhausted()); @@ -226,7 +225,7 @@ class TPartBtreeIndexIt : public IIndexIter { PushNextState(*State.back().Pos - 1); } - for (size_t level : xrange(State.size() - 1, Meta.LevelsCount)) { + for (ui32 level : xrange(State.size() - 1, Meta.LevelsCount)) { if (!TryLoad(State[level])) { // exiting with an intermediate state Y_DEBUG_ABORT_UNLESS(!IsLeaf() && !IsExhausted()); @@ -287,7 +286,7 @@ class TPartBtreeIndexIt : public IIndexIter { State[0].Pos = { }; } - for (size_t level : xrange(State.size() - 1, Meta.LevelsCount)) { + for (ui32 level : xrange(State.size() - 1, Meta.LevelsCount)) { auto &state = State[level]; Y_DEBUG_ABORT_UNLESS(seek.BelongsTo(state)); if (!TryLoad(state)) { From ba5dcfa64429ab4edc274282272ee7a372e6acae Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 26 Dec 2023 09:13:26 +0000 Subject: [PATCH 02/11] wip precharge --- .../flat_part_charge_btree_index.h | 101 ++++++++++++------ 1 file changed, 67 insertions(+), 34 deletions(-) diff --git a/ydb/core/tablet_flat/flat_part_charge_btree_index.h b/ydb/core/tablet_flat/flat_part_charge_btree_index.h index 2c802e71882b..16ff0ebf1a2c 100644 --- a/ydb/core/tablet_flat/flat_part_charge_btree_index.h +++ b/ydb/core/tablet_flat/flat_part_charge_btree_index.h @@ -6,45 +6,78 @@ namespace NKikimr::NTable { - class TChargeBTreeIndex : public ICharge { - public: - TChargeBTreeIndex(IPages *env, const TPart &part, TTagsRef tags, bool includeHistory = false) { - Y_UNUSED(env); - Y_UNUSED(part); - Y_UNUSED(tags); - Y_UNUSED(includeHistory); - } +class TChargeBTreeIndex : public ICharge { + using TBtreeIndexNode = NPage::TBtreeIndexNode; + using TBtreeIndexMeta = NPage::TBtreeIndexMeta; - TResult Do(const TCells key1, const TCells key2, const TRowId row1, - const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, - ui64 bytesLimit) const noexcept override { - // TODO: implement - Y_UNUSED(key1); - Y_UNUSED(key2); - Y_UNUSED(row1); - Y_UNUSED(row2); - Y_UNUSED(keyDefaults); - Y_UNUSED(itemsLimit); - Y_UNUSED(bytesLimit); - return {true, false}; - } +public: + TChargeBTreeIndex(IPages *env, const TPart &part, TTagsRef tags, bool includeHistory = false) + : Part(&part) + , Env(env) { + Y_UNUSED(env); + Y_UNUSED(part); + Y_UNUSED(tags); + Y_UNUSED(includeHistory); + } + +public: + TResult Do(const TCells key1, const TCells key2, const TRowId row1, + const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, + ui64 bytesLimit) const noexcept override { + // TODO: implement + Y_UNUSED(key1); + Y_UNUSED(key2); + Y_UNUSED(row1); + Y_UNUSED(row2); + Y_UNUSED(keyDefaults); + Y_UNUSED(itemsLimit); + Y_UNUSED(bytesLimit); + + auto& meta = Part->IndexPages.BTreeGroups[0]; - TResult DoReverse(const TCells key1, const TCells key2, const TRowId row1, - const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, - ui64 bytesLimit) const noexcept override { - // TODO: implement - Y_UNUSED(key1); - Y_UNUSED(key2); - Y_UNUSED(row1); - Y_UNUSED(row2); - Y_UNUSED(keyDefaults); - Y_UNUSED(itemsLimit); - Y_UNUSED(bytesLimit); - return {true, false}; + TVector level, nextLevel(Reserve(2)); + for (ui32 high : xrange(meta.LevelsCount)) { + if (high == 0) { + if (!TryLoad(meta.PageId, nextLevel)) { + return {false, false}; + } + } else { + Y_ABORT_UNLESS(level); + + + } } - private: + return {true, false}; + } + + TResult DoReverse(const TCells key1, const TCells key2, const TRowId row1, + const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, + ui64 bytesLimit) const noexcept override { + // TODO: implement + Y_UNUSED(key1); + Y_UNUSED(key2); + Y_UNUSED(row1); + Y_UNUSED(row2); + Y_UNUSED(keyDefaults); + Y_UNUSED(itemsLimit); + Y_UNUSED(bytesLimit); + return {true, false}; + } + +private: + bool TryLoad(TPageId pageId, TVector& level) const noexcept { + auto page = Env->TryGetPage(Part, pageId); + if (page) { + level.emplace_back(*page); + return true; + } + return false; + } +private: + const TPart* const Part; + IPages* const Env; }; } From 55199fe2c7748bbe27fba6df5c6b66a1ab7cf56e Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 26 Dec 2023 14:52:28 +0000 Subject: [PATCH 03/11] minor --- ydb/core/tablet_flat/flat_part_charge_btree_index.h | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/core/tablet_flat/flat_part_charge_btree_index.h b/ydb/core/tablet_flat/flat_part_charge_btree_index.h index 16ff0ebf1a2c..26081253659d 100644 --- a/ydb/core/tablet_flat/flat_part_charge_btree_index.h +++ b/ydb/core/tablet_flat/flat_part_charge_btree_index.h @@ -24,7 +24,6 @@ class TChargeBTreeIndex : public ICharge { TResult Do(const TCells key1, const TCells key2, const TRowId row1, const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override { - // TODO: implement Y_UNUSED(key1); Y_UNUSED(key2); Y_UNUSED(row1); From ea37fece585425249ce6eea8d72fc646c214be5f Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 27 Dec 2023 13:54:31 +0000 Subject: [PATCH 04/11] wip precharge rows --- ydb/core/tablet_flat/flat_page_btree_index.h | 2 +- .../flat_part_charge_btree_index.h | 60 +++++- ydb/core/tablet_flat/ut/ut_btree_index.cpp | 194 +++++++++++++++--- 3 files changed, 214 insertions(+), 42 deletions(-) diff --git a/ydb/core/tablet_flat/flat_page_btree_index.h b/ydb/core/tablet_flat/flat_page_btree_index.h index 794a517a5ad1..9699e44787e7 100644 --- a/ydb/core/tablet_flat/flat_page_btree_index.h +++ b/ydb/core/tablet_flat/flat_page_btree_index.h @@ -316,7 +316,7 @@ namespace NKikimr::NTable::NPage { return beginRowId <= rowId && rowId < endRowId; } - TRecIdx Seek(TRowId rowId, std::optional on) const noexcept + TRecIdx Seek(TRowId rowId, std::optional on = { }) const noexcept { const TRecIdx childrenCount = GetChildrenCount(); if (on && on >= childrenCount) { diff --git a/ydb/core/tablet_flat/flat_part_charge_btree_index.h b/ydb/core/tablet_flat/flat_part_charge_btree_index.h index 26081253659d..66c7ec043400 100644 --- a/ydb/core/tablet_flat/flat_part_charge_btree_index.h +++ b/ydb/core/tablet_flat/flat_part_charge_btree_index.h @@ -9,6 +9,8 @@ namespace NKikimr::NTable { class TChargeBTreeIndex : public ICharge { using TBtreeIndexNode = NPage::TBtreeIndexNode; using TBtreeIndexMeta = NPage::TBtreeIndexMeta; + using TRecIdx = NPage::TRecIdx; + using TGroupId = NPage::TGroupId; public: TChargeBTreeIndex(IPages *env, const TPart &part, TTagsRef tags, bool includeHistory = false) @@ -24,6 +26,8 @@ class TChargeBTreeIndex : public ICharge { TResult Do(const TCells key1, const TCells key2, const TRowId row1, const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override { + bool ready = true; + Y_UNUSED(key1); Y_UNUSED(key2); Y_UNUSED(row1); @@ -32,22 +36,58 @@ class TChargeBTreeIndex : public ICharge { Y_UNUSED(itemsLimit); Y_UNUSED(bytesLimit); + // TODO: row1 > row2 + // TODO: key1 > key2 + auto& meta = Part->IndexPages.BTreeGroups[0]; TVector level, nextLevel(Reserve(2)); - for (ui32 high : xrange(meta.LevelsCount)) { + for (ui32 high = 0; high < meta.LevelsCount && ready; high++) { if (high == 0) { - if (!TryLoad(meta.PageId, nextLevel)) { - return {false, false}; - } + ready &= TryLoadNode(meta.PageId, nextLevel); } else { - Y_ABORT_UNLESS(level); - + for (auto i : xrange(level.size())) { + TRecIdx begin = 0, end = level[i].GetChildrenCount(); + if (i == 0) { + begin = level[i].Seek(row1); + } + if (i + 1 == level.size()) { + end = level[i].Seek(row2) + 1; + } + for (TRecIdx j : xrange(begin, end)) { + ready &= TryLoadNode(level[i].GetShortChild(j).PageId, nextLevel); + } + } + } + + level.swap(nextLevel); + nextLevel.clear(); + } + if (!ready) { + // some index pages are missing, do not continue + return {false, false}; + } + + if (meta.LevelsCount == 0) { + // TODO: no index nodes + } else { + for (auto i : xrange(level.size())) { + TRecIdx begin = 0, end = level[i].GetChildrenCount(); + if (i == 0) { + begin = level[i].Seek(row1); + } + if (i + 1 == level.size()) { + end = level[i].Seek(row2) + 1; + } + for (TRecIdx j : xrange(begin, end)) { + ready &= HasDataPage(level[i].GetShortChild(j).PageId, { }); + } } } - return {true, false}; + // TODO: overshot for keys search + return {ready, false}; } TResult DoReverse(const TCells key1, const TCells key2, const TRowId row1, @@ -65,7 +105,11 @@ class TChargeBTreeIndex : public ICharge { } private: - bool TryLoad(TPageId pageId, TVector& level) const noexcept { + bool HasDataPage(TPageId pageId, TGroupId groupId) const noexcept { + return Env->TryGetPage(Part, pageId, groupId); + } + + bool TryLoadNode(TPageId pageId, TVector& level) const noexcept { auto page = Env->TryGetPage(Part, pageId); if (page) { level.emplace_back(*page); diff --git a/ydb/core/tablet_flat/ut/ut_btree_index.cpp b/ydb/core/tablet_flat/ut/ut_btree_index.cpp index 88bf2fb4b340..09cb362ac0c2 100644 --- a/ydb/core/tablet_flat/ut/ut_btree_index.cpp +++ b/ydb/core/tablet_flat/ut/ut_btree_index.cpp @@ -1,6 +1,8 @@ #include "flat_page_btree_index.h" #include "flat_page_btree_index_writer.h" #include "flat_part_btree_index_iter.h" +#include "flat_part_charge.h" +#include "flat_part_charge_btree_index.h" #include "test/libs/table/test_writer.h" #include #include @@ -13,15 +15,33 @@ namespace { using TChild = TBtreeIndexNode::TChild; struct TTouchEnv : public NTest::TTestEnv { - const TSharedData* TryGetPage(const TPart *part, TPageId id, TGroupId groupId) override + const TSharedData* TryGetPage(const TPart *part, TPageId pageId, TGroupId groupId) override { - UNIT_ASSERT_C(part->GetPageType(id) == EPage::BTreeIndex || part->GetPageType(id) == EPage::Index, "Shouldn't request non-index pages"); - if (!Touched[groupId].insert(id).second) { - return NTest::TTestEnv::TryGetPage(part, id, groupId); + UNIT_ASSERT_C(part->GetPageType(pageId) == EPage::BTreeIndex || part->GetPageType(pageId) == EPage::Index, "Shouldn't request non-index pages"); + Touched[groupId].insert(pageId); + if (Has[groupId].contains(pageId)) { + return NTest::TTestEnv::TryGetPage(part, pageId, groupId); } return nullptr; } + static void LoadTouched(IPages& env, bool clearHas) { + auto touchEnv = dynamic_cast(&env); + if (touchEnv) { + auto &has = touchEnv->Has; + auto &touched = touchEnv->Touched; + + if (clearHas) { + has.clear(); + } + for (const auto &g : touched) { + has[g.first].insert(g.second.begin(), g.second.end()); + } + touched.clear(); + } + } + + TMap> Has; TMap> Touched; }; @@ -670,6 +690,13 @@ Y_UNIT_TEST_SUITE(TBtreeIndexBuilder) { Y_UNIT_TEST_SUITE(TBtreeIndexTPart) { + Y_UNIT_TEST(Conf) { + NPage::TConf conf; + + // do not accidentally turn this setting on in trunk + UNIT_ASSERT_VALUES_EQUAL(conf.WriteBTreeIndex, false); + } + Y_UNIT_TEST(NoNodes) { TLayoutCook lay; @@ -898,50 +925,47 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) { } } - EReady Retry(std::function action, const TString& message, ui32 failsAllowed = 10) { + EReady Retry(std::function action, IPages& env, const TString& message, ui32 failsAllowed = 10) { while (true) { if (auto ready = action(); ready != EReady::Page) { return ready; } + TTouchEnv::LoadTouched(env, false); UNIT_ASSERT_C(failsAllowed--, "Too many fails " + message); } - return EReady::Page; + Y_UNREACHABLE(); } - template - EReady SeekRowId(TIter& iter, TRowId rowId, const TString& message, ui32 failsAllowed = 10) { + EReady SeekRowId(IIndexIter& iter, IPages& env, TRowId rowId, const TString& message, ui32 failsAllowed = 10) { return Retry([&]() { return iter.Seek(rowId); - }, message, failsAllowed); + }, env, message, failsAllowed); } - template - EReady SeekLast(TIter& iter, const TString& message, ui32 failsAllowed = 10) { + EReady SeekLast(IIndexIter& iter, IPages& env, const TString& message, ui32 failsAllowed = 10) { return Retry([&]() { return iter.SeekLast(); - }, message, failsAllowed); + }, env, message, failsAllowed); } - template - EReady SeekKey(TIter& iter, ESeek seek, bool reverse, TCells key, const TKeyCellDefaults *keyDefaults, const TString& message, ui32 failsAllowed = 10) { + EReady SeekKey(IIndexIter& iter, IPages& env, ESeek seek, bool reverse, TCells key, const TKeyCellDefaults *keyDefaults, const TString& message, ui32 failsAllowed = 10) { return Retry([&]() { if (reverse) { return iter.SeekReverse(seek, key, keyDefaults); } else { return iter.Seek(seek, key, keyDefaults); } - }, message, failsAllowed); + }, env, message, failsAllowed); } - template - EReady NextPrev(TIter& iter, bool next, const TString& message, ui32 failsAllowed = 10) { + EReady NextPrev(IIndexIter& iter, IPages& env, bool next, const TString& message, ui32 failsAllowed = 10) { return Retry([&]() { if (next) { return iter.Next(); } else { return iter.Prev(); } - }, message, failsAllowed); + }, env, message, failsAllowed); } template @@ -955,8 +979,8 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) { // checking initial seek: { TString message = TStringBuilder() << "SeekRowId<" << typeid(TEnv).name() << "> " << rowId1; - EReady bTreeReady = SeekRowId(bTree, rowId1, message); - EReady flatReady = SeekRowId(flat, rowId1, message); + EReady bTreeReady = SeekRowId(bTree, env, rowId1, message); + EReady flatReady = SeekRowId(flat, env, rowId1, message); UNIT_ASSERT_VALUES_EQUAL(bTreeReady, rowId1 < part.Stat.Rows ? EReady::Data : EReady::Gone); AssertEqual(bTree, bTreeReady, flat, flatReady, message); } @@ -964,8 +988,8 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) { // checking repositioning: { TString message = TStringBuilder() << "SeekRowId<" << typeid(TEnv).name() << "> " << rowId1 << " -> " << rowId2; - EReady bTreeReady = SeekRowId(bTree, rowId2, message); - EReady flatReady = SeekRowId(flat, rowId2, message); + EReady bTreeReady = SeekRowId(bTree, env, rowId2, message); + EReady flatReady = SeekRowId(flat, env, rowId2, message); UNIT_ASSERT_VALUES_EQUAL(bTreeReady, rowId2 < part.Stat.Rows ? EReady::Data : EReady::Gone); AssertEqual(bTree, bTreeReady, flat, flatReady, message); } @@ -980,8 +1004,8 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) { TPartIndexIt flat(&part, &env, { }); TString message = TStringBuilder() << "SeekLast<" << typeid(TEnv).name() << ">"; - EReady bTreeReady = SeekLast(bTree, message); - EReady flatReady = SeekLast(flat, message); + EReady bTreeReady = SeekLast(bTree, env, message); + EReady flatReady = SeekLast(flat, env, message); UNIT_ASSERT_VALUES_EQUAL(bTreeReady, EReady::Data); AssertEqual(bTree, bTreeReady, flat, flatReady, message); } @@ -1003,8 +1027,8 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) { message << c.AsValue() << " "; } - EReady bTreeReady = SeekKey(bTree, seek, reverse, key, keyDefaults, message); - EReady flatReady = SeekKey(flat, seek, reverse, key, keyDefaults, message); + EReady bTreeReady = SeekKey(bTree, env, seek, reverse, key, keyDefaults, message); + EReady flatReady = SeekKey(flat, env, seek, reverse, key, keyDefaults, message); UNIT_ASSERT_VALUES_EQUAL_C(bTreeReady, key.empty() ? flatReady : EReady::Data, "Can't be exhausted"); AssertEqual(bTree, bTreeReady, flat, flatReady, message, !key.empty()); @@ -1029,8 +1053,8 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) { // checking initial seek: { TString message = TStringBuilder() << "CheckNext<" << typeid(TEnv).name() << "> " << rowId; - EReady bTreeReady = SeekRowId(bTree, rowId, message); - EReady flatReady = SeekRowId(flat, rowId, message); + EReady bTreeReady = SeekRowId(bTree, env, rowId, message); + EReady flatReady = SeekRowId(flat, env, rowId, message); UNIT_ASSERT_VALUES_EQUAL(bTreeReady, rowId < part.Stat.Rows ? EReady::Data : EReady::Gone); AssertEqual(bTree, bTreeReady, flat, flatReady, message); } @@ -1039,8 +1063,8 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) { while (true) { TString message = TStringBuilder() << "CheckNext<" << typeid(TEnv).name() << "> " << rowId << " -> " << rowId; - EReady bTreeReady = NextPrev(bTree, next, message); - EReady flatReady = NextPrev(flat, next, message); + EReady bTreeReady = NextPrev(bTree, env, next, message); + EReady flatReady = NextPrev(flat, env, next, message); AssertEqual(bTree, bTreeReady, flat, flatReady, message); if (flatReady == EReady::Gone) { break; @@ -1083,11 +1107,115 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) { CheckNextPrev(part); } - Y_UNIT_TEST(Conf) { + Y_UNIT_TEST(NoNodes) { NPage::TConf conf; - // to not accidentally turn this setting on in trunk - UNIT_ASSERT_VALUES_EQUAL(conf.WriteBTreeIndex, false); + CheckPart(std::move(conf), 100, 0); + } + + Y_UNIT_TEST(OneNode) { + NPage::TConf conf; + conf.Group(0).PageRows = 2; + + CheckPart(std::move(conf), 100, 1); + } + + Y_UNIT_TEST(FewNodes) { + NPage::TConf conf; + conf.Group(0).PageRows = 2; + conf.Group(0).BTreeIndexNodeKeysMin = 3; + conf.Group(0).BTreeIndexNodeKeysMax = 4; + + CheckPart(std::move(conf), 300, 3); + } +} + +Y_UNIT_TEST_SUITE(TChargeBTreeIndex) { + void AssertEqual(const TPartStore& part, const TMap>& bTree, const TMap>& flat, const TString& message) { + TSet groupIds; + for (const auto &c : {bTree, flat}) { + for (const auto &g : c) { + groupIds.insert(g.first); + } + } + + for (TGroupId groupId : groupIds) { + TSet bTreeDataPages, flatDataPages; + for (TPageId pageId : bTree.Value(groupId, TSet{})) { + if (part.GetPageType(pageId, groupId) == EPage::DataPage) { + bTreeDataPages.insert(pageId); + } + } + for (TPageId pageId : flat.Value(groupId, TSet{})) { + if (part.GetPageType(pageId, groupId) == EPage::DataPage) { + flatDataPages.insert(pageId); + } + } + + UNIT_ASSERT_VALUES_EQUAL_C(bTreeDataPages.size(), flatDataPages.size(), message); + } + } + + void AssertEqual(const TPartStore& part, const TTouchEnv& bTree, const TTouchEnv& flat, const TString& message) { + AssertEqual(part, bTree.Has, flat.Has, message + " Has"); + AssertEqual(part, bTree.Touched, flat.Touched, message + " Touched"); + } + + void DoChargeRowId(ICharge& charge, const TRowId row1, const TRowId row2, ui64 itemsLimit, ui64 bytesLimit, + const TKeyCellDefaults &keyDefaults, const TString& message, ui32 failsAllowed = 10) { + while (true) { + if (charge.Do(row1, row2, keyDefaults, itemsLimit, bytesLimit)) { + return; + } + UNIT_ASSERT_C(failsAllowed--, "Too many fails " + message); + } + Y_UNREACHABLE(); + } + + void CheckChargeRowId(const TPartStore& part, TTagsRef tags, const TKeyCellDefaults *keyDefaults) { + for (TRowId rowId1 : xrange(part.Stat.Rows + 1)) { + for (TRowId rowId2 : xrange(part.Stat.Rows + 1)) { + TTouchEnv bTreeEnv, flatEnv; + TChargeBTreeIndex bTree(&bTreeEnv, part, tags, true); + TCharge flat(&flatEnv, part, tags, true); + + TString message = TStringBuilder() << "ChargeRowId " << rowId1 << " " << rowId2; + DoChargeRowId(bTree, rowId1, rowId2, 0, 0, *keyDefaults, message); + DoChargeRowId(bTree, rowId1, rowId2, 0, 0, *keyDefaults, message); + AssertEqual(part, bTreeEnv, flatEnv, message); + } + } + } + + void CheckPart(TConf&& conf, ui32 rows, ui32 levels) { + TLayoutCook lay; + + lay + .Col(0, 0, NScheme::NTypeIds::Uint32) + .Col(0, 1, NScheme::NTypeIds::Uint32) + .Key({0, 1}); + + conf.WriteBTreeIndex = true; + TPartCook cook(lay, conf); + + for (ui32 i : xrange(1u, rows + 1)) { + cook.Add(*TSchemedCookRow(*lay).Col(i / 7, i % 7)); + } + + TPartEggs eggs = cook.Finish(); + + const auto part = *eggs.Lone(); + + Cerr << DumpPart(part, 1) << Endl; + + UNIT_ASSERT_VALUES_EQUAL(part.IndexPages.BTreeGroups[0].LevelsCount, levels); + + auto tags = TVector(); + for (auto c : eggs.Scheme->Cols) { + tags.push_back(c.Tag); + } + + CheckChargeRowId(part, tags, eggs.Scheme->Keys.Get()); } Y_UNIT_TEST(NoNodes) { From 635ea67d7504b9684944d232da1ebdc930723323 Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 27 Dec 2023 15:03:41 +0000 Subject: [PATCH 05/11] test charge --- ydb/core/tablet_flat/ut/ut_btree_index.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ydb/core/tablet_flat/ut/ut_btree_index.cpp b/ydb/core/tablet_flat/ut/ut_btree_index.cpp index 09cb362ac0c2..71c0788b1c78 100644 --- a/ydb/core/tablet_flat/ut/ut_btree_index.cpp +++ b/ydb/core/tablet_flat/ut/ut_btree_index.cpp @@ -1157,16 +1157,17 @@ Y_UNIT_TEST_SUITE(TChargeBTreeIndex) { } void AssertEqual(const TPartStore& part, const TTouchEnv& bTree, const TTouchEnv& flat, const TString& message) { - AssertEqual(part, bTree.Has, flat.Has, message + " Has"); - AssertEqual(part, bTree.Touched, flat.Touched, message + " Touched"); + AssertEqual(part, bTree.Has, flat.Has, message); + AssertEqual(part, bTree.Touched, flat.Touched, message); } - void DoChargeRowId(ICharge& charge, const TRowId row1, const TRowId row2, ui64 itemsLimit, ui64 bytesLimit, + void DoChargeRowId(ICharge& charge, IPages& env, const TRowId row1, const TRowId row2, ui64 itemsLimit, ui64 bytesLimit, const TKeyCellDefaults &keyDefaults, const TString& message, ui32 failsAllowed = 10) { while (true) { if (charge.Do(row1, row2, keyDefaults, itemsLimit, bytesLimit)) { return; } + TTouchEnv::LoadTouched(env, false); UNIT_ASSERT_C(failsAllowed--, "Too many fails " + message); } Y_UNREACHABLE(); @@ -1180,8 +1181,8 @@ Y_UNIT_TEST_SUITE(TChargeBTreeIndex) { TCharge flat(&flatEnv, part, tags, true); TString message = TStringBuilder() << "ChargeRowId " << rowId1 << " " << rowId2; - DoChargeRowId(bTree, rowId1, rowId2, 0, 0, *keyDefaults, message); - DoChargeRowId(bTree, rowId1, rowId2, 0, 0, *keyDefaults, message); + DoChargeRowId(bTree, bTreeEnv, rowId1, rowId2, 0, 0, *keyDefaults, message); + DoChargeRowId(flat, flatEnv, rowId1, rowId2, 0, 0, *keyDefaults, message); AssertEqual(part, bTreeEnv, flatEnv, message); } } From 0b2aa250357c782d554efbe641ed378c4755aace Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 27 Dec 2023 15:50:49 +0000 Subject: [PATCH 06/11] fix typo --- ydb/core/tablet_flat/flat_part_charge.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/tablet_flat/flat_part_charge.h b/ydb/core/tablet_flat/flat_part_charge.h index 3a7e676c7792..41912e3c8cb6 100644 --- a/ydb/core/tablet_flat/flat_part_charge.h +++ b/ydb/core/tablet_flat/flat_part_charge.h @@ -252,7 +252,7 @@ namespace NTable { } } if (itemsLimit && prechargeCurrentFirstRowId <= prechargeCurrentLastRowId) { - ui64 left = itemsLimit - items; // we count only foolprof taken rows, so here we may precharge some extra rows + ui64 left = itemsLimit - items; // we count only foolproof taken rows, so here we may precharge some extra rows if (prechargeCurrentLastRowId - prechargeCurrentFirstRowId > left) { prechargeCurrentLastRowId = prechargeCurrentFirstRowId + left; } @@ -347,7 +347,7 @@ namespace NTable { } if (itemsLimit && prechargeCurrentFirstRowId >= prechargeCurrentLastRowId) { - ui64 left = itemsLimit - items; // we count only foolprof taken rows, so here we may precharge some extra rows + ui64 left = itemsLimit - items; // we count only foolproof taken rows, so here we may precharge some extra rows if (prechargeCurrentFirstRowId - prechargeCurrentLastRowId > left) { prechargeCurrentLastRowId = prechargeCurrentFirstRowId - left; } From 5d0e50f8a8c557fa271fdc4c7a351d8428d1230f Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 27 Dec 2023 16:12:25 +0000 Subject: [PATCH 07/11] one node works --- ydb/core/tablet_flat/flat_part_charge_btree_index.h | 6 +++++- ydb/core/tablet_flat/ut/ut_btree_index.cpp | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ydb/core/tablet_flat/flat_part_charge_btree_index.h b/ydb/core/tablet_flat/flat_part_charge_btree_index.h index 66c7ec043400..a4233833587f 100644 --- a/ydb/core/tablet_flat/flat_part_charge_btree_index.h +++ b/ydb/core/tablet_flat/flat_part_charge_btree_index.h @@ -41,6 +41,10 @@ class TChargeBTreeIndex : public ICharge { auto& meta = Part->IndexPages.BTreeGroups[0]; + if (Y_UNLIKELY(row1 >= meta.Count)) { + return { true, true }; // already out of bounds, nothing to precharge + } + TVector level, nextLevel(Reserve(2)); for (ui32 high = 0; high < meta.LevelsCount && ready; high++) { if (high == 0) { @@ -70,7 +74,7 @@ class TChargeBTreeIndex : public ICharge { } if (meta.LevelsCount == 0) { - // TODO: no index nodes + ready &= HasDataPage(meta.PageId, { }); } else { for (auto i : xrange(level.size())) { TRecIdx begin = 0, end = level[i].GetChildrenCount(); diff --git a/ydb/core/tablet_flat/ut/ut_btree_index.cpp b/ydb/core/tablet_flat/ut/ut_btree_index.cpp index 71c0788b1c78..9bb95c03727e 100644 --- a/ydb/core/tablet_flat/ut/ut_btree_index.cpp +++ b/ydb/core/tablet_flat/ut/ut_btree_index.cpp @@ -17,7 +17,6 @@ namespace { struct TTouchEnv : public NTest::TTestEnv { const TSharedData* TryGetPage(const TPart *part, TPageId pageId, TGroupId groupId) override { - UNIT_ASSERT_C(part->GetPageType(pageId) == EPage::BTreeIndex || part->GetPageType(pageId) == EPage::Index, "Shouldn't request non-index pages"); Touched[groupId].insert(pageId); if (Has[groupId].contains(pageId)) { return NTest::TTestEnv::TryGetPage(part, pageId, groupId); @@ -1152,7 +1151,7 @@ Y_UNIT_TEST_SUITE(TChargeBTreeIndex) { } } - UNIT_ASSERT_VALUES_EQUAL_C(bTreeDataPages.size(), flatDataPages.size(), message); + UNIT_ASSERT_VALUES_EQUAL_C(flatDataPages, bTreeDataPages, message); } } From eb9b94cc11775b14d6c0fa5606ea03f32a61997a Mon Sep 17 00:00:00 2001 From: kungasc Date: Thu, 28 Dec 2023 12:45:02 +0000 Subject: [PATCH 08/11] rows count --- ydb/core/tablet_flat/flat_page_btree_index.h | 20 ++++++------- .../flat_page_btree_index_writer.h | 20 ++++++------- .../tablet_flat/flat_part_btree_index_iter.h | 6 ++-- ydb/core/tablet_flat/flat_part_charge.h | 4 +-- .../flat_part_charge_btree_index.h | 28 ++++++++----------- ydb/core/tablet_flat/flat_part_charge_iface.h | 8 +++--- ydb/core/tablet_flat/flat_part_loader.cpp | 4 +-- ydb/core/tablet_flat/flat_part_writer.h | 4 +-- .../tablet_flat/protos/flat_table_part.proto | 4 +-- .../tablet_flat/test/libs/table/test_writer.h | 4 +-- ydb/core/tablet_flat/ut/ut_btree_index.cpp | 10 +++---- 11 files changed, 54 insertions(+), 58 deletions(-) diff --git a/ydb/core/tablet_flat/flat_page_btree_index.h b/ydb/core/tablet_flat/flat_page_btree_index.h index 9699e44787e7..54c94909dd26 100644 --- a/ydb/core/tablet_flat/flat_page_btree_index.h +++ b/ydb/core/tablet_flat/flat_page_btree_index.h @@ -85,7 +85,7 @@ namespace NKikimr::NTable::NPage { struct TShortChild { TPageId PageId; - TRowId Count; + TRowId RowsCount; ui64 DataSize; auto operator<=>(const TShortChild&) const = default; @@ -95,22 +95,22 @@ namespace NKikimr::NTable::NPage { struct TChild { TPageId PageId; - TRowId Count; + TRowId RowsCount; ui64 DataSize; - TRowId ErasedCount; + TRowId ErasedRowsCount; auto operator<=>(const TChild&) const = default; TString ToString() const noexcept { - return TStringBuilder() << "PageId: " << PageId << " Count: " << Count << " DataSize: " << DataSize << " Erased: " << ErasedCount; + return TStringBuilder() << "PageId: " << PageId << " RowsCount: " << RowsCount << " DataSize: " << DataSize << " ErasedRowsCount: " << ErasedRowsCount; } } Y_PACKED; static_assert(sizeof(TChild) == 28, "Invalid TBtreeIndexNode TChild size"); static_assert(offsetof(TChild, PageId) == offsetof(TShortChild, PageId)); - static_assert(offsetof(TChild, Count) == offsetof(TShortChild, Count)); + static_assert(offsetof(TChild, RowsCount) == offsetof(TShortChild, RowsCount)); static_assert(offsetof(TChild, DataSize) == offsetof(TShortChild, DataSize)); #pragma pack(pop) @@ -306,7 +306,7 @@ namespace NKikimr::NTable::NPage { { if (Header->IsShortChildFormat) { const TShortChild* const shortChild = TDeref::At(Children, pos * sizeof(TShortChild)); - return { shortChild->PageId, shortChild->Count, shortChild->DataSize, 0 }; + return { shortChild->PageId, shortChild->RowsCount, shortChild->DataSize, 0 }; } else { return *TDeref::At(Children, pos * sizeof(TChild)); } @@ -326,19 +326,19 @@ namespace NKikimr::NTable::NPage { auto range = xrange(0u, childrenCount); const auto cmp = [this](TRowId rowId, TPos pos) { - return rowId < GetShortChild(pos).Count; + return rowId < GetShortChild(pos).RowsCount; }; TRecIdx result; if (!on) { // Will do a full binary search on full range - } else if (GetShortChild(*on).Count <= rowId) { + } else if (GetShortChild(*on).RowsCount <= rowId) { // Try a short linear search first result = *on; for (int linear = 0; linear < 4; ++linear) { result++; Y_ABORT_UNLESS(result < childrenCount, "Should always seek some child"); - if (GetShortChild(result).Count > rowId) { + if (GetShortChild(result).RowsCount > rowId) { return result; } } @@ -352,7 +352,7 @@ namespace NKikimr::NTable::NPage { if (result == 0) { return 0; } - if (GetShortChild(result - 1).Count <= rowId) { + if (GetShortChild(result - 1).RowsCount <= rowId) { return result; } result--; diff --git a/ydb/core/tablet_flat/flat_page_btree_index_writer.h b/ydb/core/tablet_flat/flat_page_btree_index_writer.h index bef4b93ceb71..ee8ad40278cf 100644 --- a/ydb/core/tablet_flat/flat_page_btree_index_writer.h +++ b/ydb/core/tablet_flat/flat_page_btree_index_writer.h @@ -50,7 +50,7 @@ namespace NKikimr::NTable::NPage { } void AddChild(TChild child) { - Y_ABORT_UNLESS(child.ErasedCount == 0 || !IsShortChildFormat(), "Short format can't have ErasedCount"); + Y_ABORT_UNLESS(child.ErasedRowsCount == 0 || !IsShortChildFormat(), "Short format can't have ErasedRowsCount"); Children.push_back(child); } @@ -249,7 +249,7 @@ namespace NKikimr::NTable::NPage { void PlaceChild(const TChild& child) noexcept { if (IsShortChildFormat()) { - Place() = TShortChild{child.PageId, child.Count, child.DataSize}; + Place() = TShortChild{child.PageId, child.RowsCount, child.DataSize}; } else { Place() = child; } @@ -375,14 +375,14 @@ namespace NKikimr::NTable::NPage { } void AddShortChild(TShortChild child) { - AddChild(TChild{child.PageId, child.Count, child.DataSize, 0}); + AddChild(TChild{child.PageId, child.RowsCount, child.DataSize, 0}); } void AddChild(TChild child) { // aggregate in order to perform search by row id from any leaf node - child.Count = (ChildrenCount += child.Count); + child.RowsCount = (ChildrenRowsCount += child.RowsCount); child.DataSize = (ChildrenSize += child.DataSize); - child.ErasedCount = (ChildrenErasedCount += child.ErasedCount); + child.ErasedRowsCount = (ChildrenErasedRowsCount += child.ErasedRowsCount); Levels[0].PushChild(child); } @@ -409,8 +409,8 @@ namespace NKikimr::NTable::NPage { IndexSize = 0; Writer.Reset(); Levels = { TLevel() }; - ChildrenCount = 0; - ChildrenErasedCount = 0; + ChildrenRowsCount = 0; + ChildrenErasedRowsCount = 0; ChildrenSize = 0; } @@ -463,7 +463,7 @@ namespace NKikimr::NTable::NPage { if (levelIndex + 1 == Levels.size()) { Levels.emplace_back(); } - Levels[levelIndex + 1].PushChild(TChild{pageId, lastChild.Count, lastChild.DataSize, lastChild.ErasedCount}); + Levels[levelIndex + 1].PushChild(TChild{pageId, lastChild.RowsCount, lastChild.DataSize, lastChild.ErasedRowsCount}); if (!last) { Levels[levelIndex + 1].PushKey(Levels[levelIndex].PopKey()); } @@ -498,8 +498,8 @@ namespace NKikimr::NTable::NPage { const ui32 NodeKeysMin; const ui32 NodeKeysMax; - TRowId ChildrenCount = 0; - TRowId ChildrenErasedCount = 0; + TRowId ChildrenRowsCount = 0; + TRowId ChildrenErasedRowsCount = 0; ui64 ChildrenSize = 0; }; diff --git a/ydb/core/tablet_flat/flat_part_btree_index_iter.h b/ydb/core/tablet_flat/flat_part_btree_index_iter.h index df10bc376426..2d1617d64e56 100644 --- a/ydb/core/tablet_flat/flat_part_btree_index_iter.h +++ b/ydb/core/tablet_flat/flat_part_btree_index_iter.h @@ -246,7 +246,7 @@ class TPartBtreeIndexIt : public IIndexIter { } TRowId GetEndRowId() const override { - return Meta.Count; + return Meta.RowsCount; } TPageId GetPageId() const override { @@ -334,8 +334,8 @@ class TPartBtreeIndexIt : public IIndexIter { auto child = current.Node->GetChild(pos); - TRowId beginRowId = pos ? current.Node->GetChild(pos - 1).Count : current.BeginRowId; - TRowId endRowId = child.Count; + TRowId beginRowId = pos ? current.Node->GetChild(pos - 1).RowsCount : current.BeginRowId; + TRowId endRowId = child.RowsCount; TCellsIterable beginKey = pos ? current.Node->GetKeyCellsIterable(pos - 1, GroupInfo.ColsKeyIdx) : current.BeginKey; TCellsIterable endKey = pos < current.Node->GetKeysCount() ? current.Node->GetKeyCellsIterable(pos, GroupInfo.ColsKeyIdx) : current.EndKey; diff --git a/ydb/core/tablet_flat/flat_part_charge.h b/ydb/core/tablet_flat/flat_part_charge.h index 41912e3c8cb6..a4a3cfbed490 100644 --- a/ydb/core/tablet_flat/flat_part_charge.h +++ b/ydb/core/tablet_flat/flat_part_charge.h @@ -42,7 +42,7 @@ namespace NTable { } } - TResult Do(const TCells key1, const TCells key2, const TRowId row1, const TRowId row2, + TResult Do(const TCells key1, const TCells key2, TRowId row1, TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override { auto index = Index.TryLoadRaw(); @@ -110,7 +110,7 @@ namespace NTable { return { ready, overshot }; } - TResult DoReverse(const TCells key1, const TCells key2, const TRowId row1, const TRowId row2, + TResult DoReverse(const TCells key1, const TCells key2, TRowId row1, TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override { auto index = Index.TryLoadRaw(); diff --git a/ydb/core/tablet_flat/flat_part_charge_btree_index.h b/ydb/core/tablet_flat/flat_part_charge_btree_index.h index a4233833587f..2d91aceb346a 100644 --- a/ydb/core/tablet_flat/flat_part_charge_btree_index.h +++ b/ydb/core/tablet_flat/flat_part_charge_btree_index.h @@ -23,39 +23,36 @@ class TChargeBTreeIndex : public ICharge { } public: - TResult Do(const TCells key1, const TCells key2, const TRowId row1, - const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, - ui64 bytesLimit) const noexcept override { + TResult Do(const TCells key1, const TCells key2, TRowId row1, TRowId row2, + const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override { bool ready = true; Y_UNUSED(key1); Y_UNUSED(key2); - Y_UNUSED(row1); - Y_UNUSED(row2); Y_UNUSED(keyDefaults); Y_UNUSED(itemsLimit); Y_UNUSED(bytesLimit); - // TODO: row1 > row2 - // TODO: key1 > key2 - auto& meta = Part->IndexPages.BTreeGroups[0]; - if (Y_UNLIKELY(row1 >= meta.Count)) { + if (Y_UNLIKELY(row1 >= meta.RowsCount)) { return { true, true }; // already out of bounds, nothing to precharge } + if (Y_UNLIKELY(row1 > row2)) { + row2 = row1; // will not go further than row1 + } TVector level, nextLevel(Reserve(2)); for (ui32 high = 0; high < meta.LevelsCount && ready; high++) { if (high == 0) { ready &= TryLoadNode(meta.PageId, nextLevel); } else { - for (auto i : xrange(level.size())) { + for (ui32 i : xrange(level.size())) { TRecIdx begin = 0, end = level[i].GetChildrenCount(); if (i == 0) { begin = level[i].Seek(row1); } - if (i + 1 == level.size()) { + if (i + 1 == level.size() && row2 < meta.RowsCount) { end = level[i].Seek(row2) + 1; } for (TRecIdx j : xrange(begin, end)) { @@ -76,12 +73,12 @@ class TChargeBTreeIndex : public ICharge { if (meta.LevelsCount == 0) { ready &= HasDataPage(meta.PageId, { }); } else { - for (auto i : xrange(level.size())) { + for (ui32 i : xrange(level.size())) { TRecIdx begin = 0, end = level[i].GetChildrenCount(); if (i == 0) { begin = level[i].Seek(row1); } - if (i + 1 == level.size()) { + if (i + 1 == level.size() && row2 < meta.RowsCount) { end = level[i].Seek(row2) + 1; } for (TRecIdx j : xrange(begin, end)) { @@ -94,9 +91,8 @@ class TChargeBTreeIndex : public ICharge { return {ready, false}; } - TResult DoReverse(const TCells key1, const TCells key2, const TRowId row1, - const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, - ui64 bytesLimit) const noexcept override { + TResult DoReverse(const TCells key1, const TCells key2, TRowId row1, TRowId row2, + const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override { // TODO: implement Y_UNUSED(key1); Y_UNUSED(key2); diff --git a/ydb/core/tablet_flat/flat_part_charge_iface.h b/ydb/core/tablet_flat/flat_part_charge_iface.h index 46e84175d6b0..126b99e0b66d 100644 --- a/ydb/core/tablet_flat/flat_part_charge_iface.h +++ b/ydb/core/tablet_flat/flat_part_charge_iface.h @@ -17,7 +17,7 @@ namespace NKikimr::NTable { * * Important caveat: assumes iteration won't touch any row > row2 */ - bool Do(const TRowId row1, const TRowId row2, + bool Do(TRowId row1, TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept { return Do(TCells{}, TCells{}, row1, row2, @@ -29,7 +29,7 @@ namespace NKikimr::NTable { * * Important caveat: assumes iteration won't touch any row > row2 */ - bool DoReverse(const TRowId row1, const TRowId row2, + bool DoReverse(TRowId row1, TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept { return DoReverse(TCells{}, TCells{}, row1, row2, @@ -39,13 +39,13 @@ namespace NKikimr::NTable { /** * Precharges data for rows between max(key1, row1) and min(key2, row2) inclusive */ - virtual TResult Do(const TCells key1, const TCells key2, const TRowId row1, const TRowId row2, + virtual TResult Do(const TCells key1, const TCells key2, TRowId row1, TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept = 0; /** * Precharges data for rows between min(key1, row1) and max(key2, row2) inclusive in reverse */ - virtual TResult DoReverse(const TCells key1, const TCells key2, const TRowId row1, const TRowId row2, + virtual TResult DoReverse(const TCells key1, const TCells key2, TRowId row1, TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept = 0; virtual ~ICharge() = default; diff --git a/ydb/core/tablet_flat/flat_part_loader.cpp b/ydb/core/tablet_flat/flat_part_loader.cpp index fcf6820fdeca..6b1eaa6b1578 100644 --- a/ydb/core/tablet_flat/flat_part_loader.cpp +++ b/ydb/core/tablet_flat/flat_part_loader.cpp @@ -83,9 +83,9 @@ void TLoader::StageParseMeta() noexcept for (const auto &meta : history ? layout.GetBTreeHistoricIndexes() : layout.GetBTreeGroupIndexes()) { NPage::TBtreeIndexMeta converted{{ meta.GetRootPageId(), - meta.GetCount(), + meta.GetRowsCount(), meta.GetDataSize(), - meta.GetErasedCount()}, + meta.GetErasedRowsCount()}, meta.GetLevelsCount(), meta.GetIndexSize()}; (history ? BTreeHistoricIndexes : BTreeGroupIndexes).push_back(converted); diff --git a/ydb/core/tablet_flat/flat_part_writer.h b/ydb/core/tablet_flat/flat_part_writer.h index e35c2979b0bb..6d08ad9cfc37 100644 --- a/ydb/core/tablet_flat/flat_part_writer.h +++ b/ydb/core/tablet_flat/flat_part_writer.h @@ -669,8 +669,8 @@ namespace NTable { m->SetLevelsCount(meta.LevelsCount); m->SetIndexSize(meta.IndexSize); m->SetDataSize(meta.DataSize); - m->SetCount(meta.Count); - m->SetErasedCount(meta.ErasedCount); + m->SetRowsCount(meta.RowsCount); + m->SetErasedRowsCount(meta.ErasedRowsCount); } } diff --git a/ydb/core/tablet_flat/protos/flat_table_part.proto b/ydb/core/tablet_flat/protos/flat_table_part.proto index 4155ca9006ae..2ff6c4468035 100644 --- a/ydb/core/tablet_flat/protos/flat_table_part.proto +++ b/ydb/core/tablet_flat/protos/flat_table_part.proto @@ -28,8 +28,8 @@ message TBTreeIndexMeta { optional uint32 LevelsCount = 2; optional uint64 IndexSize = 3; optional uint64 DataSize = 4; - optional uint64 Count = 5; // Total number of data rows - optional uint64 ErasedCount = 6; // Total number of erased data rows + optional uint64 RowsCount = 5; // Total number of data rows + optional uint64 ErasedRowsCount = 6; // Total number of erased data rows } message TLayout { diff --git a/ydb/core/tablet_flat/test/libs/table/test_writer.h b/ydb/core/tablet_flat/test/libs/table/test_writer.h index 4a4936470850..cc6f897e17d3 100644 --- a/ydb/core/tablet_flat/test/libs/table/test_writer.h +++ b/ydb/core/tablet_flat/test/libs/table/test_writer.h @@ -129,9 +129,9 @@ namespace NTest { for (const auto &meta : history ? lay.GetBTreeHistoricIndexes() : lay.GetBTreeGroupIndexes()) { NPage::TBtreeIndexMeta converted{{ meta.GetRootPageId(), - meta.GetCount(), + meta.GetRowsCount(), meta.GetDataSize(), - meta.GetErasedCount()}, + meta.GetErasedRowsCount()}, meta.GetLevelsCount(), meta.GetIndexSize()}; (history ? BTreeHistoricIndexes : BTreeGroupIndexes).push_back(converted); diff --git a/ydb/core/tablet_flat/ut/ut_btree_index.cpp b/ydb/core/tablet_flat/ut/ut_btree_index.cpp index 9bb95c03727e..8045c4613d37 100644 --- a/ydb/core/tablet_flat/ut/ut_btree_index.cpp +++ b/ydb/core/tablet_flat/ut/ut_btree_index.cpp @@ -174,7 +174,7 @@ namespace { UNIT_ASSERT_VALUES_EQUAL(node.GetKeysCount() + 1, children.size()); for (TRecIdx i : xrange(node.GetKeysCount() + 1)) { UNIT_ASSERT_EQUAL(node.GetChild(i), children[i]); - TShortChild shortChild{children[i].PageId, children[i].Count, children[i].DataSize}; + TShortChild shortChild{children[i].PageId, children[i].RowsCount, children[i].DataSize}; UNIT_ASSERT_EQUAL(node.GetShortChild(i), shortChild); } } @@ -314,7 +314,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexNode) { writer.AddKey(deserialized.GetCells()); } for (auto &c : children) { - c.ErasedCount = 0; + c.ErasedRowsCount = 0; writer.AddChild(c); } @@ -359,7 +359,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexNode) { writer.AddKey(deserialized.GetCells()); } for (auto &c : children) { - c.ErasedCount = 0; + c.ErasedRowsCount = 0; writer.AddChild(c); } @@ -644,9 +644,9 @@ Y_UNIT_TEST_SUITE(TBtreeIndexBuilder) { TBtreeIndexMeta expected{{9, 0, 0, 0}, 3, 1550}; for (auto c : children) { - expected.Count += c.Count; + expected.RowsCount += c.RowsCount; expected.DataSize += c.DataSize; - expected.ErasedCount += c.ErasedCount; + expected.ErasedRowsCount += c.ErasedRowsCount; } UNIT_ASSERT_EQUAL_C(*result, expected, "Got " + result->ToString()); } From 78d7a3b567318624bfe9b2ad8d022987ba2fb203 Mon Sep 17 00:00:00 2001 From: kungasc Date: Thu, 28 Dec 2023 15:29:14 +0000 Subject: [PATCH 09/11] reverse --- .../flat_part_charge_btree_index.h | 80 ++++++++++++++++--- ydb/core/tablet_flat/ut/ut_btree_index.cpp | 18 +++-- 2 files changed, 79 insertions(+), 19 deletions(-) diff --git a/ydb/core/tablet_flat/flat_part_charge_btree_index.h b/ydb/core/tablet_flat/flat_part_charge_btree_index.h index 2d91aceb346a..2c81ee35ca24 100644 --- a/ydb/core/tablet_flat/flat_part_charge_btree_index.h +++ b/ydb/core/tablet_flat/flat_part_charge_btree_index.h @@ -48,14 +48,14 @@ class TChargeBTreeIndex : public ICharge { ready &= TryLoadNode(meta.PageId, nextLevel); } else { for (ui32 i : xrange(level.size())) { - TRecIdx begin = 0, end = level[i].GetChildrenCount(); + TRecIdx from = 0, to = level[i].GetKeysCount(); if (i == 0) { - begin = level[i].Seek(row1); + from = level[i].Seek(row1); } if (i + 1 == level.size() && row2 < meta.RowsCount) { - end = level[i].Seek(row2) + 1; + to = level[i].Seek(row2); } - for (TRecIdx j : xrange(begin, end)) { + for (TRecIdx j : xrange(from, to + 1)) { ready &= TryLoadNode(level[i].GetShortChild(j).PageId, nextLevel); } } @@ -74,14 +74,14 @@ class TChargeBTreeIndex : public ICharge { ready &= HasDataPage(meta.PageId, { }); } else { for (ui32 i : xrange(level.size())) { - TRecIdx begin = 0, end = level[i].GetChildrenCount(); + TRecIdx from = 0, to = level[i].GetKeysCount(); if (i == 0) { - begin = level[i].Seek(row1); + from = level[i].Seek(row1); } if (i + 1 == level.size() && row2 < meta.RowsCount) { - end = level[i].Seek(row2) + 1; + to = level[i].Seek(row2); } - for (TRecIdx j : xrange(begin, end)) { + for (TRecIdx j : xrange(from, to + 1)) { ready &= HasDataPage(level[i].GetShortChild(j).PageId, { }); } } @@ -93,15 +93,71 @@ class TChargeBTreeIndex : public ICharge { TResult DoReverse(const TCells key1, const TCells key2, TRowId row1, TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override { - // TODO: implement + bool ready = true; + Y_UNUSED(key1); Y_UNUSED(key2); - Y_UNUSED(row1); - Y_UNUSED(row2); Y_UNUSED(keyDefaults); Y_UNUSED(itemsLimit); Y_UNUSED(bytesLimit); - return {true, false}; + + auto& meta = Part->IndexPages.BTreeGroups[0]; + + if (Y_UNLIKELY(row1 >= meta.RowsCount)) { + row1 = meta.RowsCount - 1; // start from the last row + } + if (Y_UNLIKELY(row1 < row2)) { + row2 = row1; // will not go further than row1 + } + + // level contains nodes in reverse order + TVector level, nextLevel(Reserve(2)); + for (ui32 high = 0; high < meta.LevelsCount && ready; high++) { + if (high == 0) { + ready &= TryLoadNode(meta.PageId, nextLevel); + } else { + for (ui32 i : xrange(level.size())) { + TRecIdx from = level[i].GetKeysCount(), to = 0; + if (i == 0) { + from = level[i].Seek(row1); + } + if (i + 1 == level.size() && row2 < meta.RowsCount) { + to = level[i].Seek(row2); + } + for (TRecIdx j = from + 1; j > to; j--) { + ready &= TryLoadNode(level[i].GetShortChild(j - 1).PageId, nextLevel); + } + } + } + + level.swap(nextLevel); + nextLevel.clear(); + } + + if (!ready) { + // some index pages are missing, do not continue + return {false, false}; + } + + if (meta.LevelsCount == 0) { + ready &= HasDataPage(meta.PageId, { }); + } else { + for (ui32 i : xrange(level.size())) { + TRecIdx from = level[i].GetKeysCount(), to = 0; + if (i == 0) { + from = level[i].Seek(row1); + } + if (i + 1 == level.size() && row2 < meta.RowsCount) { + to = level[i].Seek(row2); + } + for (TRecIdx j = from + 1; j > to; j--) { + ready &= HasDataPage(level[i].GetShortChild(j - 1).PageId, { }); + } + } + } + + // TODO: overshot for keys search + return {ready, false}; } private: diff --git a/ydb/core/tablet_flat/ut/ut_btree_index.cpp b/ydb/core/tablet_flat/ut/ut_btree_index.cpp index 8045c4613d37..5b44aa5daa26 100644 --- a/ydb/core/tablet_flat/ut/ut_btree_index.cpp +++ b/ydb/core/tablet_flat/ut/ut_btree_index.cpp @@ -1161,9 +1161,12 @@ Y_UNIT_TEST_SUITE(TChargeBTreeIndex) { } void DoChargeRowId(ICharge& charge, IPages& env, const TRowId row1, const TRowId row2, ui64 itemsLimit, ui64 bytesLimit, - const TKeyCellDefaults &keyDefaults, const TString& message, ui32 failsAllowed = 10) { + bool reverse, const TKeyCellDefaults &keyDefaults, const TString& message, ui32 failsAllowed = 10) { while (true) { - if (charge.Do(row1, row2, keyDefaults, itemsLimit, bytesLimit)) { + bool ready = reverse + ? charge.DoReverse(row1, row2, keyDefaults, itemsLimit, bytesLimit) + : charge.Do(row1, row2, keyDefaults, itemsLimit, bytesLimit); + if (ready) { return; } TTouchEnv::LoadTouched(env, false); @@ -1172,16 +1175,16 @@ Y_UNIT_TEST_SUITE(TChargeBTreeIndex) { Y_UNREACHABLE(); } - void CheckChargeRowId(const TPartStore& part, TTagsRef tags, const TKeyCellDefaults *keyDefaults) { + void CheckChargeRowId(const TPartStore& part, TTagsRef tags, const TKeyCellDefaults *keyDefaults, bool reverse) { for (TRowId rowId1 : xrange(part.Stat.Rows + 1)) { for (TRowId rowId2 : xrange(part.Stat.Rows + 1)) { TTouchEnv bTreeEnv, flatEnv; TChargeBTreeIndex bTree(&bTreeEnv, part, tags, true); TCharge flat(&flatEnv, part, tags, true); - TString message = TStringBuilder() << "ChargeRowId " << rowId1 << " " << rowId2; - DoChargeRowId(bTree, bTreeEnv, rowId1, rowId2, 0, 0, *keyDefaults, message); - DoChargeRowId(flat, flatEnv, rowId1, rowId2, 0, 0, *keyDefaults, message); + TString message = TStringBuilder() << (reverse ? "ChargeRowIdReverse " : "ChargeRowId ") << rowId1 << " " << rowId2; + DoChargeRowId(bTree, bTreeEnv, rowId1, rowId2, 0, 0, reverse, *keyDefaults, message); + DoChargeRowId(flat, flatEnv, rowId1, rowId2, 0, 0, reverse, *keyDefaults, message); AssertEqual(part, bTreeEnv, flatEnv, message); } } @@ -1215,7 +1218,8 @@ Y_UNIT_TEST_SUITE(TChargeBTreeIndex) { tags.push_back(c.Tag); } - CheckChargeRowId(part, tags, eggs.Scheme->Keys.Get()); + CheckChargeRowId(part, tags, eggs.Scheme->Keys.Get(), false); + CheckChargeRowId(part, tags, eggs.Scheme->Keys.Get(), true); } Y_UNIT_TEST(NoNodes) { From 5be885ac4adfd6531bc192471687f9cb89eb4356 Mon Sep 17 00:00:00 2001 From: kungasc Date: Thu, 28 Dec 2023 16:46:38 +0000 Subject: [PATCH 10/11] fix style --- ydb/core/tablet_flat/flat_part_charge_btree_index.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/tablet_flat/flat_part_charge_btree_index.h b/ydb/core/tablet_flat/flat_part_charge_btree_index.h index 2c81ee35ca24..1574b1c04019 100644 --- a/ydb/core/tablet_flat/flat_part_charge_btree_index.h +++ b/ydb/core/tablet_flat/flat_part_charge_btree_index.h @@ -33,7 +33,7 @@ class TChargeBTreeIndex : public ICharge { Y_UNUSED(itemsLimit); Y_UNUSED(bytesLimit); - auto& meta = Part->IndexPages.BTreeGroups[0]; + const auto& meta = Part->IndexPages.BTreeGroups[0]; if (Y_UNLIKELY(row1 >= meta.RowsCount)) { return { true, true }; // already out of bounds, nothing to precharge @@ -101,7 +101,7 @@ class TChargeBTreeIndex : public ICharge { Y_UNUSED(itemsLimit); Y_UNUSED(bytesLimit); - auto& meta = Part->IndexPages.BTreeGroups[0]; + const auto& meta = Part->IndexPages.BTreeGroups[0]; if (Y_UNLIKELY(row1 >= meta.RowsCount)) { row1 = meta.RowsCount - 1; // start from the last row From b6238779ef0ba3a082e5753206eba046f926bc6c Mon Sep 17 00:00:00 2001 From: kungasc Date: Fri, 29 Dec 2023 15:39:45 +0000 Subject: [PATCH 11/11] cr: fix grammar --- .../tablet_flat/benchmark/b_part_index.cpp | 2 +- ydb/core/tablet_flat/flat_page_btree_index.h | 24 ++++++++--------- .../flat_page_btree_index_writer.h | 26 +++++++++---------- .../tablet_flat/flat_part_btree_index_iter.h | 20 +++++++------- .../flat_part_charge_btree_index.h | 26 +++++++++---------- ydb/core/tablet_flat/flat_part_dump.cpp | 2 +- ydb/core/tablet_flat/flat_part_loader.cpp | 6 ++--- ydb/core/tablet_flat/flat_part_writer.h | 6 ++--- .../tablet_flat/protos/flat_table_part.proto | 6 ++--- .../tablet_flat/test/libs/table/test_writer.h | 6 ++--- ydb/core/tablet_flat/ut/ut_btree_index.cpp | 24 ++++++++--------- ydb/core/tablet_flat/ut/ut_part.cpp | 2 +- .../ut_large/ut_btree_index_large.cpp | 12 ++++----- 13 files changed, 81 insertions(+), 81 deletions(-) diff --git a/ydb/core/tablet_flat/benchmark/b_part_index.cpp b/ydb/core/tablet_flat/benchmark/b_part_index.cpp index bdc875909192..c991a2d13fff 100644 --- a/ydb/core/tablet_flat/benchmark/b_part_index.cpp +++ b/ydb/core/tablet_flat/benchmark/b_part_index.cpp @@ -70,7 +70,7 @@ namespace { Cerr << "DataBytes = " << part->Stat.Bytes << " DataPages = " << IndexTools::CountMainPages(*part) << Endl; Cerr << "FlatIndexBytes = " << part->GetPageSize(part->IndexPages.Groups[groups ? 1 : 0], {}) << " BTreeIndexBytes = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].IndexSize << Endl; - Cerr << "Levels = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].LevelsCount << Endl; + Cerr << "Levels = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].LevelCount << Endl; // 100 MB UNIT_ASSERT_GE(part->Stat.Bytes, 100ull*1024*1024); diff --git a/ydb/core/tablet_flat/flat_page_btree_index.h b/ydb/core/tablet_flat/flat_page_btree_index.h index 54c94909dd26..0fc722dcce92 100644 --- a/ydb/core/tablet_flat/flat_page_btree_index.h +++ b/ydb/core/tablet_flat/flat_page_btree_index.h @@ -85,7 +85,7 @@ namespace NKikimr::NTable::NPage { struct TShortChild { TPageId PageId; - TRowId RowsCount; + TRowId RowCount; ui64 DataSize; auto operator<=>(const TShortChild&) const = default; @@ -95,22 +95,22 @@ namespace NKikimr::NTable::NPage { struct TChild { TPageId PageId; - TRowId RowsCount; + TRowId RowCount; ui64 DataSize; - TRowId ErasedRowsCount; + TRowId ErasedRowCount; auto operator<=>(const TChild&) const = default; TString ToString() const noexcept { - return TStringBuilder() << "PageId: " << PageId << " RowsCount: " << RowsCount << " DataSize: " << DataSize << " ErasedRowsCount: " << ErasedRowsCount; + return TStringBuilder() << "PageId: " << PageId << " RowCount: " << RowCount << " DataSize: " << DataSize << " ErasedRowCount: " << ErasedRowCount; } } Y_PACKED; static_assert(sizeof(TChild) == 28, "Invalid TBtreeIndexNode TChild size"); static_assert(offsetof(TChild, PageId) == offsetof(TShortChild, PageId)); - static_assert(offsetof(TChild, RowsCount) == offsetof(TShortChild, RowsCount)); + static_assert(offsetof(TChild, RowCount) == offsetof(TShortChild, RowCount)); static_assert(offsetof(TChild, DataSize) == offsetof(TShortChild, DataSize)); #pragma pack(pop) @@ -306,7 +306,7 @@ namespace NKikimr::NTable::NPage { { if (Header->IsShortChildFormat) { const TShortChild* const shortChild = TDeref::At(Children, pos * sizeof(TShortChild)); - return { shortChild->PageId, shortChild->RowsCount, shortChild->DataSize, 0 }; + return { shortChild->PageId, shortChild->RowCount, shortChild->DataSize, 0 }; } else { return *TDeref::At(Children, pos * sizeof(TChild)); } @@ -326,19 +326,19 @@ namespace NKikimr::NTable::NPage { auto range = xrange(0u, childrenCount); const auto cmp = [this](TRowId rowId, TPos pos) { - return rowId < GetShortChild(pos).RowsCount; + return rowId < GetShortChild(pos).RowCount; }; TRecIdx result; if (!on) { // Will do a full binary search on full range - } else if (GetShortChild(*on).RowsCount <= rowId) { + } else if (GetShortChild(*on).RowCount <= rowId) { // Try a short linear search first result = *on; for (int linear = 0; linear < 4; ++linear) { result++; Y_ABORT_UNLESS(result < childrenCount, "Should always seek some child"); - if (GetShortChild(result).RowsCount > rowId) { + if (GetShortChild(result).RowCount > rowId) { return result; } } @@ -352,7 +352,7 @@ namespace NKikimr::NTable::NPage { if (result == 0) { return 0; } - if (GetShortChild(result - 1).RowsCount <= rowId) { + if (GetShortChild(result - 1).RowCount <= rowId) { return result; } result--; @@ -464,14 +464,14 @@ namespace NKikimr::NTable::NPage { }; struct TBtreeIndexMeta : public TBtreeIndexNode::TChild { - ui32 LevelsCount; + ui32 LevelCount; ui64 IndexSize; auto operator<=>(const TBtreeIndexMeta&) const = default; TString ToString() const noexcept { - return TStringBuilder() << TBtreeIndexNode::TChild::ToString() << " LevelsCount: " << LevelsCount << " IndexSize: " << IndexSize; + return TStringBuilder() << TBtreeIndexNode::TChild::ToString() << " LevelCount: " << LevelCount << " IndexSize: " << IndexSize; } }; } diff --git a/ydb/core/tablet_flat/flat_page_btree_index_writer.h b/ydb/core/tablet_flat/flat_page_btree_index_writer.h index ee8ad40278cf..2a4627bf7e2d 100644 --- a/ydb/core/tablet_flat/flat_page_btree_index_writer.h +++ b/ydb/core/tablet_flat/flat_page_btree_index_writer.h @@ -50,7 +50,7 @@ namespace NKikimr::NTable::NPage { } void AddChild(TChild child) { - Y_ABORT_UNLESS(child.ErasedRowsCount == 0 || !IsShortChildFormat(), "Short format can't have ErasedRowsCount"); + Y_ABORT_UNLESS(child.ErasedRowCount == 0 || !IsShortChildFormat(), "Short format can't have ErasedRowCount"); Children.push_back(child); } @@ -249,7 +249,7 @@ namespace NKikimr::NTable::NPage { void PlaceChild(const TChild& child) noexcept { if (IsShortChildFormat()) { - Place() = TShortChild{child.PageId, child.RowsCount, child.DataSize}; + Place() = TShortChild{child.PageId, child.RowCount, child.DataSize}; } else { Place() = child; } @@ -375,14 +375,14 @@ namespace NKikimr::NTable::NPage { } void AddShortChild(TShortChild child) { - AddChild(TChild{child.PageId, child.RowsCount, child.DataSize, 0}); + AddChild(TChild{child.PageId, child.RowCount, child.DataSize, 0}); } void AddChild(TChild child) { // aggregate in order to perform search by row id from any leaf node - child.RowsCount = (ChildrenRowsCount += child.RowsCount); - child.DataSize = (ChildrenSize += child.DataSize); - child.ErasedRowsCount = (ChildrenErasedRowsCount += child.ErasedRowsCount); + child.RowCount = (ChildRowCount += child.RowCount); + child.DataSize = (ChildSize += child.DataSize); + child.ErasedRowCount = (ChildErasedRowCount += child.ErasedRowCount); Levels[0].PushChild(child); } @@ -409,9 +409,9 @@ namespace NKikimr::NTable::NPage { IndexSize = 0; Writer.Reset(); Levels = { TLevel() }; - ChildrenRowsCount = 0; - ChildrenErasedRowsCount = 0; - ChildrenSize = 0; + ChildRowCount = 0; + ChildErasedRowCount = 0; + ChildSize = 0; } private: @@ -463,7 +463,7 @@ namespace NKikimr::NTable::NPage { if (levelIndex + 1 == Levels.size()) { Levels.emplace_back(); } - Levels[levelIndex + 1].PushChild(TChild{pageId, lastChild.RowsCount, lastChild.DataSize, lastChild.ErasedRowsCount}); + Levels[levelIndex + 1].PushChild(TChild{pageId, lastChild.RowCount, lastChild.DataSize, lastChild.ErasedRowCount}); if (!last) { Levels[levelIndex + 1].PushKey(Levels[levelIndex].PopKey()); } @@ -498,9 +498,9 @@ namespace NKikimr::NTable::NPage { const ui32 NodeKeysMin; const ui32 NodeKeysMax; - TRowId ChildrenRowsCount = 0; - TRowId ChildrenErasedRowsCount = 0; - ui64 ChildrenSize = 0; + TRowId ChildRowCount = 0; + TRowId ChildErasedRowCount = 0; + ui64 ChildSize = 0; }; } diff --git a/ydb/core/tablet_flat/flat_part_btree_index_iter.h b/ydb/core/tablet_flat/flat_part_btree_index_iter.h index 2d1617d64e56..d924e670de40 100644 --- a/ydb/core/tablet_flat/flat_part_btree_index_iter.h +++ b/ydb/core/tablet_flat/flat_part_btree_index_iter.h @@ -115,7 +115,7 @@ class TPartBtreeIndexIt : public IIndexIter { , GroupId(groupId) , GroupInfo(part->Scheme->GetLayout(groupId)) , Meta(groupId.IsHistoric() ? part->IndexPages.BTreeHistoric[groupId.Index] : part->IndexPages.BTreeGroups[groupId.Index]) - , State(Reserve(Meta.LevelsCount + 1)) + , State(Reserve(Meta.LevelCount + 1)) { const static TCellsIterable EmptyKey(static_cast(nullptr), TColumns()); State.emplace_back(Meta, 0, GetEndRowId(), EmptyKey, EmptyKey); @@ -180,7 +180,7 @@ class TPartBtreeIndexIt : public IIndexIter { EReady Next() override { Y_ABORT_UNLESS(!IsExhausted()); - if (Meta.LevelsCount == 0) { + if (Meta.LevelCount == 0) { return Exhaust(); } @@ -194,7 +194,7 @@ class TPartBtreeIndexIt : public IIndexIter { PushNextState(*State.back().Pos + 1); } - for (ui32 level : xrange(State.size() - 1, Meta.LevelsCount)) { + for (ui32 level : xrange(State.size() - 1, Meta.LevelCount)) { if (!TryLoad(State[level])) { // exiting with an intermediate state Y_DEBUG_ABORT_UNLESS(!IsLeaf() && !IsExhausted()); @@ -211,7 +211,7 @@ class TPartBtreeIndexIt : public IIndexIter { EReady Prev() override { Y_ABORT_UNLESS(!IsExhausted()); - if (Meta.LevelsCount == 0) { + if (Meta.LevelCount == 0) { return Exhaust(); } @@ -225,7 +225,7 @@ class TPartBtreeIndexIt : public IIndexIter { PushNextState(*State.back().Pos - 1); } - for (ui32 level : xrange(State.size() - 1, Meta.LevelsCount)) { + for (ui32 level : xrange(State.size() - 1, Meta.LevelCount)) { if (!TryLoad(State[level])) { // exiting with an intermediate state Y_DEBUG_ABORT_UNLESS(!IsLeaf() && !IsExhausted()); @@ -246,7 +246,7 @@ class TPartBtreeIndexIt : public IIndexIter { } TRowId GetEndRowId() const override { - return Meta.RowsCount; + return Meta.RowCount; } TPageId GetPageId() const override { @@ -286,7 +286,7 @@ class TPartBtreeIndexIt : public IIndexIter { State[0].Pos = { }; } - for (ui32 level : xrange(State.size() - 1, Meta.LevelsCount)) { + for (ui32 level : xrange(State.size() - 1, Meta.LevelCount)) { auto &state = State[level]; Y_DEBUG_ABORT_UNLESS(seek.BelongsTo(state)); if (!TryLoad(state)) { @@ -316,7 +316,7 @@ class TPartBtreeIndexIt : public IIndexIter { bool IsLeaf() const noexcept { // Note: it is possible to have 0 levels in B-Tree // so we may have exhausted state with leaf (data) node - return State.size() == Meta.LevelsCount + 1 && !IsExhausted(); + return State.size() == Meta.LevelCount + 1 && !IsExhausted(); } EReady Exhaust() { @@ -334,8 +334,8 @@ class TPartBtreeIndexIt : public IIndexIter { auto child = current.Node->GetChild(pos); - TRowId beginRowId = pos ? current.Node->GetChild(pos - 1).RowsCount : current.BeginRowId; - TRowId endRowId = child.RowsCount; + TRowId beginRowId = pos ? current.Node->GetChild(pos - 1).RowCount : current.BeginRowId; + TRowId endRowId = child.RowCount; TCellsIterable beginKey = pos ? current.Node->GetKeyCellsIterable(pos - 1, GroupInfo.ColsKeyIdx) : current.BeginKey; TCellsIterable endKey = pos < current.Node->GetKeysCount() ? current.Node->GetKeyCellsIterable(pos, GroupInfo.ColsKeyIdx) : current.EndKey; diff --git a/ydb/core/tablet_flat/flat_part_charge_btree_index.h b/ydb/core/tablet_flat/flat_part_charge_btree_index.h index 1574b1c04019..5876cc2eea65 100644 --- a/ydb/core/tablet_flat/flat_part_charge_btree_index.h +++ b/ydb/core/tablet_flat/flat_part_charge_btree_index.h @@ -35,7 +35,7 @@ class TChargeBTreeIndex : public ICharge { const auto& meta = Part->IndexPages.BTreeGroups[0]; - if (Y_UNLIKELY(row1 >= meta.RowsCount)) { + if (Y_UNLIKELY(row1 >= meta.RowCount)) { return { true, true }; // already out of bounds, nothing to precharge } if (Y_UNLIKELY(row1 > row2)) { @@ -43,8 +43,8 @@ class TChargeBTreeIndex : public ICharge { } TVector level, nextLevel(Reserve(2)); - for (ui32 high = 0; high < meta.LevelsCount && ready; high++) { - if (high == 0) { + for (ui32 height = 0; height < meta.LevelCount && ready; height++) { + if (height == 0) { ready &= TryLoadNode(meta.PageId, nextLevel); } else { for (ui32 i : xrange(level.size())) { @@ -52,7 +52,7 @@ class TChargeBTreeIndex : public ICharge { if (i == 0) { from = level[i].Seek(row1); } - if (i + 1 == level.size() && row2 < meta.RowsCount) { + if (i + 1 == level.size() && row2 < meta.RowCount) { to = level[i].Seek(row2); } for (TRecIdx j : xrange(from, to + 1)) { @@ -70,7 +70,7 @@ class TChargeBTreeIndex : public ICharge { return {false, false}; } - if (meta.LevelsCount == 0) { + if (meta.LevelCount == 0) { ready &= HasDataPage(meta.PageId, { }); } else { for (ui32 i : xrange(level.size())) { @@ -78,7 +78,7 @@ class TChargeBTreeIndex : public ICharge { if (i == 0) { from = level[i].Seek(row1); } - if (i + 1 == level.size() && row2 < meta.RowsCount) { + if (i + 1 == level.size() && row2 < meta.RowCount) { to = level[i].Seek(row2); } for (TRecIdx j : xrange(from, to + 1)) { @@ -103,8 +103,8 @@ class TChargeBTreeIndex : public ICharge { const auto& meta = Part->IndexPages.BTreeGroups[0]; - if (Y_UNLIKELY(row1 >= meta.RowsCount)) { - row1 = meta.RowsCount - 1; // start from the last row + if (Y_UNLIKELY(row1 >= meta.RowCount)) { + row1 = meta.RowCount - 1; // start from the last row } if (Y_UNLIKELY(row1 < row2)) { row2 = row1; // will not go further than row1 @@ -112,8 +112,8 @@ class TChargeBTreeIndex : public ICharge { // level contains nodes in reverse order TVector level, nextLevel(Reserve(2)); - for (ui32 high = 0; high < meta.LevelsCount && ready; high++) { - if (high == 0) { + for (ui32 height = 0; height < meta.LevelCount && ready; height++) { + if (height == 0) { ready &= TryLoadNode(meta.PageId, nextLevel); } else { for (ui32 i : xrange(level.size())) { @@ -121,7 +121,7 @@ class TChargeBTreeIndex : public ICharge { if (i == 0) { from = level[i].Seek(row1); } - if (i + 1 == level.size() && row2 < meta.RowsCount) { + if (i + 1 == level.size() && row2 < meta.RowCount) { to = level[i].Seek(row2); } for (TRecIdx j = from + 1; j > to; j--) { @@ -139,7 +139,7 @@ class TChargeBTreeIndex : public ICharge { return {false, false}; } - if (meta.LevelsCount == 0) { + if (meta.LevelCount == 0) { ready &= HasDataPage(meta.PageId, { }); } else { for (ui32 i : xrange(level.size())) { @@ -147,7 +147,7 @@ class TChargeBTreeIndex : public ICharge { if (i == 0) { from = level[i].Seek(row1); } - if (i + 1 == level.size() && row2 < meta.RowsCount) { + if (i + 1 == level.size() && row2 < meta.RowCount) { to = level[i].Seek(row2); } for (TRecIdx j = from + 1; j > to; j--) { diff --git a/ydb/core/tablet_flat/flat_part_dump.cpp b/ydb/core/tablet_flat/flat_part_dump.cpp index 976fe8fbc2ba..d7a83f037305 100644 --- a/ydb/core/tablet_flat/flat_part_dump.cpp +++ b/ydb/core/tablet_flat/flat_part_dump.cpp @@ -168,7 +168,7 @@ namespace { { if (part.IndexPages.BTreeGroups) { auto meta = part.IndexPages.BTreeGroups.front(); - if (meta.LevelsCount) { + if (meta.LevelCount) { BTreeIndexNode(part, meta); } else { Out diff --git a/ydb/core/tablet_flat/flat_part_loader.cpp b/ydb/core/tablet_flat/flat_part_loader.cpp index 6b1eaa6b1578..294e2be31516 100644 --- a/ydb/core/tablet_flat/flat_part_loader.cpp +++ b/ydb/core/tablet_flat/flat_part_loader.cpp @@ -83,10 +83,10 @@ void TLoader::StageParseMeta() noexcept for (const auto &meta : history ? layout.GetBTreeHistoricIndexes() : layout.GetBTreeGroupIndexes()) { NPage::TBtreeIndexMeta converted{{ meta.GetRootPageId(), - meta.GetRowsCount(), + meta.GetRowCount(), meta.GetDataSize(), - meta.GetErasedRowsCount()}, - meta.GetLevelsCount(), + meta.GetErasedRowCount()}, + meta.GetLevelCount(), meta.GetIndexSize()}; (history ? BTreeHistoricIndexes : BTreeGroupIndexes).push_back(converted); } diff --git a/ydb/core/tablet_flat/flat_part_writer.h b/ydb/core/tablet_flat/flat_part_writer.h index 6d08ad9cfc37..572f1e8d54fc 100644 --- a/ydb/core/tablet_flat/flat_part_writer.h +++ b/ydb/core/tablet_flat/flat_part_writer.h @@ -666,11 +666,11 @@ namespace NTable { for (auto meta : history ? Current.BTreeHistoricIndexes : Current.BTreeGroupIndexes) { auto m = history ? lay->AddBTreeHistoricIndexes() : lay->AddBTreeGroupIndexes(); m->SetRootPageId(meta.PageId); - m->SetLevelsCount(meta.LevelsCount); + m->SetLevelCount(meta.LevelCount); m->SetIndexSize(meta.IndexSize); m->SetDataSize(meta.DataSize); - m->SetRowsCount(meta.RowsCount); - m->SetErasedRowsCount(meta.ErasedRowsCount); + m->SetRowCount(meta.RowCount); + m->SetErasedRowCount(meta.ErasedRowCount); } } diff --git a/ydb/core/tablet_flat/protos/flat_table_part.proto b/ydb/core/tablet_flat/protos/flat_table_part.proto index 2ff6c4468035..00efe95b59f8 100644 --- a/ydb/core/tablet_flat/protos/flat_table_part.proto +++ b/ydb/core/tablet_flat/protos/flat_table_part.proto @@ -25,11 +25,11 @@ message TPartScheme { message TBTreeIndexMeta { optional uint32 RootPageId = 1; - optional uint32 LevelsCount = 2; + optional uint32 LevelCount = 2; optional uint64 IndexSize = 3; optional uint64 DataSize = 4; - optional uint64 RowsCount = 5; // Total number of data rows - optional uint64 ErasedRowsCount = 6; // Total number of erased data rows + optional uint64 RowCount = 5; // Total number of data rows + optional uint64 ErasedRowCount = 6; // Total number of erased data rows } message TLayout { diff --git a/ydb/core/tablet_flat/test/libs/table/test_writer.h b/ydb/core/tablet_flat/test/libs/table/test_writer.h index cc6f897e17d3..5ed43de879d2 100644 --- a/ydb/core/tablet_flat/test/libs/table/test_writer.h +++ b/ydb/core/tablet_flat/test/libs/table/test_writer.h @@ -129,10 +129,10 @@ namespace NTest { for (const auto &meta : history ? lay.GetBTreeHistoricIndexes() : lay.GetBTreeGroupIndexes()) { NPage::TBtreeIndexMeta converted{{ meta.GetRootPageId(), - meta.GetRowsCount(), + meta.GetRowCount(), meta.GetDataSize(), - meta.GetErasedRowsCount()}, - meta.GetLevelsCount(), + meta.GetErasedRowCount()}, + meta.GetLevelCount(), meta.GetIndexSize()}; (history ? BTreeHistoricIndexes : BTreeGroupIndexes).push_back(converted); } diff --git a/ydb/core/tablet_flat/ut/ut_btree_index.cpp b/ydb/core/tablet_flat/ut/ut_btree_index.cpp index 5b44aa5daa26..1ef4a8923013 100644 --- a/ydb/core/tablet_flat/ut/ut_btree_index.cpp +++ b/ydb/core/tablet_flat/ut/ut_btree_index.cpp @@ -18,7 +18,7 @@ namespace { const TSharedData* TryGetPage(const TPart *part, TPageId pageId, TGroupId groupId) override { Touched[groupId].insert(pageId); - if (Has[groupId].contains(pageId)) { + if (Loaded[groupId].contains(pageId)) { return NTest::TTestEnv::TryGetPage(part, pageId, groupId); } return nullptr; @@ -27,7 +27,7 @@ namespace { static void LoadTouched(IPages& env, bool clearHas) { auto touchEnv = dynamic_cast(&env); if (touchEnv) { - auto &has = touchEnv->Has; + auto &has = touchEnv->Loaded; auto &touched = touchEnv->Touched; if (clearHas) { @@ -40,7 +40,7 @@ namespace { } } - TMap> Has; + TMap> Loaded; TMap> Touched; }; @@ -174,7 +174,7 @@ namespace { UNIT_ASSERT_VALUES_EQUAL(node.GetKeysCount() + 1, children.size()); for (TRecIdx i : xrange(node.GetKeysCount() + 1)) { UNIT_ASSERT_EQUAL(node.GetChild(i), children[i]); - TShortChild shortChild{children[i].PageId, children[i].RowsCount, children[i].DataSize}; + TShortChild shortChild{children[i].PageId, children[i].RowCount, children[i].DataSize}; UNIT_ASSERT_EQUAL(node.GetShortChild(i), shortChild); } } @@ -314,7 +314,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexNode) { writer.AddKey(deserialized.GetCells()); } for (auto &c : children) { - c.ErasedRowsCount = 0; + c.ErasedRowCount = 0; writer.AddChild(c); } @@ -359,7 +359,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexNode) { writer.AddKey(deserialized.GetCells()); } for (auto &c : children) { - c.ErasedRowsCount = 0; + c.ErasedRowCount = 0; writer.AddChild(c); } @@ -594,7 +594,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexBuilder) { Dump(*result, builder.GroupInfo, pager.Back()); - UNIT_ASSERT_VALUES_EQUAL(result->LevelsCount, 3); + UNIT_ASSERT_VALUES_EQUAL(result->LevelCount, 3); auto checkKeys = [&](TPageId pageId, const TVector& keys) { CheckKeys(pageId, keys, builder.GroupInfo, pager.Back()); @@ -644,9 +644,9 @@ Y_UNIT_TEST_SUITE(TBtreeIndexBuilder) { TBtreeIndexMeta expected{{9, 0, 0, 0}, 3, 1550}; for (auto c : children) { - expected.RowsCount += c.RowsCount; + expected.RowCount += c.RowCount; expected.DataSize += c.DataSize; - expected.ErasedRowsCount += c.ErasedRowsCount; + expected.ErasedRowCount += c.ErasedRowCount; } UNIT_ASSERT_EQUAL_C(*result, expected, "Got " + result->ToString()); } @@ -1094,7 +1094,7 @@ Y_UNIT_TEST_SUITE(TPartBtreeIndexIt) { Cerr << DumpPart(part, 1) << Endl; - UNIT_ASSERT_VALUES_EQUAL(part.IndexPages.BTreeGroups[0].LevelsCount, levels); + UNIT_ASSERT_VALUES_EQUAL(part.IndexPages.BTreeGroups[0].LevelCount, levels); CheckSeekRowId(part); CheckSeekRowId(part); @@ -1156,7 +1156,7 @@ Y_UNIT_TEST_SUITE(TChargeBTreeIndex) { } void AssertEqual(const TPartStore& part, const TTouchEnv& bTree, const TTouchEnv& flat, const TString& message) { - AssertEqual(part, bTree.Has, flat.Has, message); + AssertEqual(part, bTree.Loaded, flat.Loaded, message); AssertEqual(part, bTree.Touched, flat.Touched, message); } @@ -1211,7 +1211,7 @@ Y_UNIT_TEST_SUITE(TChargeBTreeIndex) { Cerr << DumpPart(part, 1) << Endl; - UNIT_ASSERT_VALUES_EQUAL(part.IndexPages.BTreeGroups[0].LevelsCount, levels); + UNIT_ASSERT_VALUES_EQUAL(part.IndexPages.BTreeGroups[0].LevelCount, levels); auto tags = TVector(); for (auto c : eggs.Scheme->Cols) { diff --git a/ydb/core/tablet_flat/ut/ut_part.cpp b/ydb/core/tablet_flat/ut/ut_part.cpp index 9be2c733299a..9d5c21c44758 100644 --- a/ydb/core/tablet_flat/ut/ut_part.cpp +++ b/ydb/core/tablet_flat/ut/ut_part.cpp @@ -440,7 +440,7 @@ Y_UNIT_TEST_SUITE(TPart) { { /*_ Ensure that B-Tree index has enough layers */ if (part.IndexPages.BTreeGroups.size()) { - UNIT_ASSERT_VALUES_EQUAL(part.IndexPages.BTreeGroups[0].LevelsCount, 3); + UNIT_ASSERT_VALUES_EQUAL(part.IndexPages.BTreeGroups[0].LevelCount, 3); } } diff --git a/ydb/core/tablet_flat/ut_large/ut_btree_index_large.cpp b/ydb/core/tablet_flat/ut_large/ut_btree_index_large.cpp index 00df27c990dc..3af2bfb805a0 100644 --- a/ydb/core/tablet_flat/ut_large/ut_btree_index_large.cpp +++ b/ydb/core/tablet_flat/ut_large/ut_btree_index_large.cpp @@ -39,7 +39,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexTPartLarge) { UNIT_ASSERT_GE(part->Stat.Bytes, 1ull*1024*1024*1024); UNIT_ASSERT_LE(part->Stat.Bytes, 1ull*1024*1024*1024 + 100*1024*1024); - UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[0].LevelsCount, 3); + UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[0].LevelCount, 3); } Y_UNIT_TEST(MiddleKeys1GB) { @@ -69,7 +69,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexTPartLarge) { UNIT_ASSERT_GE(part->Stat.Bytes, 1ull*1024*1024*1024); UNIT_ASSERT_LE(part->Stat.Bytes, 1ull*1024*1024*1024 + 100*1024*1024); - UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[0].LevelsCount, 3); + UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[0].LevelCount, 3); } Y_UNIT_TEST(BigKeys1GB) { @@ -99,7 +99,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexTPartLarge) { UNIT_ASSERT_GE(part->Stat.Bytes, 1ull*1024*1024*1024); UNIT_ASSERT_LE(part->Stat.Bytes, 1ull*1024*1024*1024 + 100*1024*1024); - UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[0].LevelsCount, 6); + UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[0].LevelCount, 6); } Y_UNIT_TEST(CutKeys) { @@ -131,7 +131,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexTPartLarge) { UNIT_ASSERT_GE(part->Stat.Bytes, 1ull*1024*1024*1024); UNIT_ASSERT_LE(part->Stat.Bytes, 1ull*1024*1024*1024 + 100*1024*1024); - UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[0].LevelsCount, 3); + UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[0].LevelCount, 3); } Y_UNIT_TEST(Group) { @@ -162,7 +162,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexTPartLarge) { UNIT_ASSERT_GE(part->Stat.Bytes, 1ull*1024*1024*1024); UNIT_ASSERT_LE(part->Stat.Bytes, 1ull*1024*1024*1024 + 100*1024*1024); - UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[1].LevelsCount, 2); + UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeGroups[1].LevelCount, 2); } Y_UNIT_TEST(History) { @@ -193,7 +193,7 @@ Y_UNIT_TEST_SUITE(TBtreeIndexTPartLarge) { UNIT_ASSERT_GE(part->Stat.Bytes, 1ull*1024*1024*1024); UNIT_ASSERT_LE(part->Stat.Bytes, 1ull*1024*1024*1024 + 100*1024*1024); - UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeHistoric[0].LevelsCount, 3); + UNIT_ASSERT_VALUES_EQUAL(part->IndexPages.BTreeHistoric[0].LevelCount, 3); } }