KIKIMR-19139 Load index in BuildStats

Author: kunga

ydb/core/tablet_flat/flat_part_index_iter.h

@@ -89,7 +89,7 @@ class TPartIndexIt {
         return DataOrGone();
     }
 
-    bool IsValid() {
+    bool IsValid() const {
         return bool(Iter);
     }
 
@@ -99,38 +99,38 @@ class TPartIndexIt {
     }
 
 public:
-    TRowId GetEndRowId() {
+    TRowId GetEndRowId() const {
         return EndRowId;
     }
 
-    TPageId GetPageId() {
+    TPageId GetPageId() const {
         Y_VERIFY(Index);
         Y_VERIFY(Iter);
         return Iter->GetPageId();
     }
 
-    TRowId GetRowId() {
+    TRowId GetRowId() const {
         Y_VERIFY(Index);
         Y_VERIFY(Iter);
         return Iter->GetRowId();
     }
 
-    TRowId GetNextRowId() {
+    TRowId GetNextRowId() const {
         Y_VERIFY(Index);
         auto next = Iter + 1;
         return next
             ? next->GetRowId()
             : Max<TRowId>();
     }
 
-    const TRecord * GetRecord() {
+    const TRecord * GetRecord() const {
         Y_VERIFY(Index);
         Y_VERIFY(Iter);
         return Iter.GetRecord();
     }
 
 private:
-    EReady DataOrGone() {
+    EReady DataOrGone() const {
         return Iter ? EReady::Data : EReady::Gone;
     }
 

ydb/core/tablet_flat/flat_stat_part.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "flat_part_iface.h"
+#include "flat_part_index_iter.h"
 #include "flat_part_laid.h"
 #include "flat_page_frames.h"
 #include "util_basics.h"
@@ -118,47 +119,69 @@ class TPartDataSizeHelper {
 // if page start key is not screened then the whole previous page is added to stats
 class TScreenedPartIndexIterator {
 public:
-    TScreenedPartIndexIterator(TPartView partView, TIntrusiveConstPtr<TKeyCellDefaults> keyColumns, 
+    TScreenedPartIndexIterator(TPartView partView, IPages* env, TIntrusiveConstPtr<TKeyCellDefaults> keyColumns, 
             TIntrusiveConstPtr<NPage::TFrames> small, TIntrusiveConstPtr<NPage::TFrames> large)
         : Part(std::move(partView.Part))
+        , Pos(Part.Get(), env, {})
         , KeyColumns(std::move(keyColumns))
         , Screen(std::move(partView.Screen))
         , Small(std::move(small))
         , Large(std::move(large))
         , CurrentHole(TScreen::Iter(Screen, CurrentHoleIdx, 0, 1))
     {
-        Pos = Part->Index->Begin();
-        End = Part->Index->End();
         AltGroups.reserve(Part->GroupsCount - 1);
         for (ui32 group : xrange(size_t(1), Part->GroupsCount)) {
-            AltGroups.emplace_back(Part.Get(), NPage::TGroupId(group));
+            AltGroups.emplace_back(Part.Get(), env, NPage::TGroupId(group));
         }
-        for (ui32 group : xrange(Part->HistoricGroupsCount)) {
-            HistoryGroups.emplace_back(Part.Get(), NPage::TGroupId(group, true));
+        for (ui32 group : xrange(Part->HistoricIndexes.size())) {
+            HistoryGroups.emplace_back(Part.Get(), env, NPage::TGroupId(group, true));
         }
+    }
+
+    EReady Start() {
+        auto ready = Pos.Seek(0);
         FillKey();
+
+        for (auto& g : AltGroups) {
+            if (g.Pos.Seek(0) == EReady::Page) {
+                ready = EReady::Page;
+            }
+        }
+        for (auto& g : HistoryGroups) {
+            if (g.Pos.Seek(0) == EReady::Page) {
+                ready = EReady::Page;
+            }
+        }
+
+        return ready;
     }
 
     bool IsValid() const {
-        return Pos != End;
+        return Pos.IsValid();
     }
 
-    void Next(TPartDataStats& stats) {
+    EReady Next(TPartDataStats& stats) {
         Y_VERIFY(IsValid());
 
-        auto curPageId = Pos->GetPageId();
-        LastRowId = Pos->GetRowId();
-        ++Pos;
+        auto curPageId = Pos.GetPageId();
+        LastRowId = Pos.GetRowId();
+        auto ready = Pos.Next();
+        if (ready == EReady::Page) {
+            return ready;
+        }
         ui64 rowCount = IncludedRows(GetLastRowId(), GetCurrentRowId());
         stats.RowCount += rowCount;
 
         if (rowCount) AddPageSize(stats.DataSize, curPageId);
-        TRowId nextRowId = Pos ? Pos->GetRowId() : Max<TRowId>();
+        TRowId nextRowId = ready == EReady::Data ? Pos.GetRowId() : Max<TRowId>();
         for (auto& g : AltGroups) {
-            while (g.Pos && g.Pos->GetRowId() < nextRowId) {
+            while (g.Pos.IsValid() && g.Pos.GetRowId() < nextRowId) {
                 // eagerly include all data up to the next row id
-                if (rowCount) AddPageSize(stats.DataSize, g.Pos->GetPageId(), g.GroupId);
-                ++g.Pos;
+                if (rowCount) AddPageSize(stats.DataSize, g.Pos.GetPageId(), g.GroupId);
+                if (g.Pos.Next() == EReady::Page) {
+                    ready = EReady::Page;
+                    break;
+                }
             }
         }
 
@@ -167,18 +190,24 @@ class TScreenedPartIndexIterator {
             auto& h = HistoryGroups[0];
             const auto& hscheme = Part->Scheme->HistoryGroup;
             Y_VERIFY_DEBUG(hscheme.ColsKeyIdx.size() == 3);
-            while (h.Pos && h.Pos->Cell(hscheme.ColsKeyIdx[0]).AsValue<TRowId>() < nextRowId) {
+            while (h.Pos.IsValid() && h.Pos.GetRecord()->Cell(hscheme.ColsKeyIdx[0]).AsValue<TRowId>() < nextRowId) {
                 // eagerly include all history up to the next row id
-                if (rowCount) AddPageSize(stats.DataSize, h.Pos->GetPageId(), h.GroupId);
-                ++h.Pos;
+                if (rowCount) AddPageSize(stats.DataSize, h.Pos.GetPageId(), h.GroupId);
+                if (h.Pos.Next() == EReady::Page) {
+                    ready = EReady::Page;
+                    break;
+                }
             }
-            TRowId nextHistoryRowId = h.Pos ? h.Pos->GetRowId() : Max<TRowId>();
+            TRowId nextHistoryRowId = h.Pos.IsValid() ? h.Pos.GetRowId() : Max<TRowId>();
             for (size_t index = 1; index < HistoryGroups.size(); ++index) {
                 auto& g = HistoryGroups[index];
-                while (g.Pos && g.Pos->GetRowId() < nextHistoryRowId) {
+                while (g.Pos.IsValid() && g.Pos.GetRowId() < nextHistoryRowId) {
                     // eagerly include all data up to the next row id
-                    if (rowCount) AddPageSize(stats.DataSize, g.Pos->GetPageId(), g.GroupId);
-                    ++g.Pos;
+                    if (rowCount) AddPageSize(stats.DataSize, g.Pos.GetPageId(), g.GroupId);
+                    if (g.Pos.Next() == EReady::Page) {
+                        ready = EReady::Page;
+                        break;
+                    }
                 }
             }
         }
@@ -193,6 +222,7 @@ class TScreenedPartIndexIterator {
         }
 
         FillKey();
+        return ready;
     }
 
     TDbTupleRef GetCurrentKey() const {
@@ -207,7 +237,7 @@ class TScreenedPartIndexIterator {
 
     ui64 GetCurrentRowId() const {
         if (IsValid()) {
-            return Pos->GetRowId();
+            return Pos.GetRowId();
         }
         if (TRowId endRowId = Part->Index.GetEndRowId(); endRowId != Max<TRowId>()) {
             // This would include the last page rows when known
@@ -233,7 +263,7 @@ class TScreenedPartIndexIterator {
         ui32 keyIdx = 0;
         // Add columns that are present in the part
         for (;keyIdx < Part->Scheme->Groups[0].KeyTypes.size(); ++keyIdx) {
-            CurrentKey.push_back(Pos->Cell(Part->Scheme->Groups[0].ColsKeyIdx[keyIdx]));
+            CurrentKey.push_back(Pos.GetRecord()->Cell(Part->Scheme->Groups[0].ColsKeyIdx[keyIdx]));
         }
 
         // Extend with default values if needed
@@ -293,20 +323,19 @@ class TScreenedPartIndexIterator {
 
 private:
     struct TGroupState {
-        NPage::TIndex::TIter Pos;
+        TPartIndexIt Pos;
         const NPage::TGroupId GroupId;
 
-        TGroupState(const TPart* part, NPage::TGroupId groupId)
-            : Pos(part->GetGroupIndex(groupId)->Begin())
+        TGroupState(const TPart* part, IPages* env, NPage::TGroupId groupId)
+            : Pos(part, env, groupId)
             , GroupId(groupId)
         { }
     };
 
 private:
     TIntrusiveConstPtr<TPart> Part;
+    TPartIndexIt Pos;
     TIntrusiveConstPtr<TKeyCellDefaults> KeyColumns;
-    NPage::TIndex::TIter Pos;
-    NPage::TIndex::TIter End;
     TSmallVec<TCell> CurrentKey;
     ui64 LastRowId = 0;
     TSmallVec<TGroupState> AltGroups;

ydb/core/tablet_flat/flat_stat_table.cpp

@@ -6,26 +6,38 @@
 namespace NKikimr {
 namespace NTable {
 
-void BuildStats(const TSubset& subset, TStats& stats, ui64 rowCountResolution, ui64 dataSizeResolution, const IPages* env) {
-    Y_UNUSED(env);
-
+bool BuildStats(const TSubset& subset, TStats& stats, ui64 rowCountResolution, ui64 dataSizeResolution, IPages* env) {
     stats.Clear();
 
     TPartDataStats stIterStats = { };
     TStatsIterator stIter(subset.Scheme->Keys);
 
     // Make index iterators for all parts
+    bool started = true;
     for (auto& pi : subset.Flatten) {
         stats.IndexSize.Add(pi->IndexesRawSize, pi->Label.Channel());
-        TAutoPtr<TScreenedPartIndexIterator> iter = new TScreenedPartIndexIterator(pi, subset.Scheme->Keys, pi->Small, pi->Large);
-        if (iter->IsValid()) {
+        TAutoPtr<TScreenedPartIndexIterator> iter = new TScreenedPartIndexIterator(pi, env, subset.Scheme->Keys, pi->Small, pi->Large);
+        auto ready = iter->Start();
+        if (ready == EReady::Page) {
+            started = false;
+        } else if (ready == EReady::Data) {
             stIter.Add(iter);
         }
     }
+    if (!started) {
+        return false;
+    }
 
     ui64 prevRows = 0;
     ui64 prevSize = 0;
-    while (stIter.Next(stIterStats)) {
+    while (true) {
+        auto ready = stIter.Next(stIterStats);
+        if (ready == EReady::Page) {
+            return false;
+        } else if (ready == EReady::Gone) {
+            break;
+        }
+
         const bool nextRowsBucket = (stIterStats.RowCount >= prevRows + rowCountResolution);
         const bool nextSizeBucket = (stIterStats.DataSize.Size >= prevSize + dataSizeResolution);
 
@@ -48,6 +60,8 @@ void BuildStats(const TSubset& subset, TStats& stats, ui64 rowCountResolution, u
 
     stats.RowCount = stIterStats.RowCount;
     stats.DataSize = std::move(stIterStats.DataSize);
+
+    return true;
 }
 
 void GetPartOwners(const TSubset& subset, THashSet<ui64>& partOwners) {

ydb/core/tablet_flat/flat_stat_table.h

@@ -25,11 +25,7 @@ class TStatsIterator {
         Heap.push(it);
     }
 
-    /**
-     * @return true when we haven't reached the end and have current key
-     * @return false when we have reached the end and don't have current key
-     */
-    bool Next(TPartDataStats& stats) {
+    EReady Next(TPartDataStats& stats) {
         ui64 lastRowCount = stats.RowCount;
         ui64 lastDataSize = stats.DataSize.Size;
 
@@ -41,24 +37,32 @@ class TStatsIterator {
             TSerializedCellVec serialized = TSerializedCellVec(TSerializedCellVec::Serialize({it->GetCurrentKey().Columns, it->GetCurrentKey().ColumnCount}));
             TDbTupleRef key(KeyColumns->BasicTypes().data(), serialized.GetCells().data(), serialized.GetCells().size());
 
-            if (MoveIterator(it, stats))
+            auto ready = it->Next(stats);
+            if (ready == EReady::Page) {
+                return ready;
+            } else if (ready == EReady::Data) {
                 Heap.push(it);
+            }
 
             // guarantees that all results will be different
             while (!Heap.empty() && CompareKeys(key, Heap.top()->GetCurrentKey()) == 0) {
                 it = Heap.top();
                 Heap.pop();
 
-                if (MoveIterator(it, stats))
+                ready = it->Next(stats);
+                if (ready == EReady::Page) {
+                    return ready;
+                } else if (ready == EReady::Data) {
                     Heap.push(it);
+                }
             }
 
             if (stats.RowCount != lastRowCount && stats.DataSize.Size != lastDataSize) {
                 break;
             }
         }
 
-        return !Heap.empty();
+        return Heap.empty() ? EReady::Gone : EReady::Data;
     }
 
     TDbTupleRef GetCurrentKey() const {
@@ -79,11 +83,6 @@ class TStatsIterator {
         }
     };
 
-    bool MoveIterator(TScreenedPartIndexIterator* it, TPartDataStats& stats) {
-        it->Next(stats);
-        return it->IsValid();
-    }
-
     TIntrusiveConstPtr<TKeyCellDefaults> KeyColumns;
     THolderVector<TScreenedPartIndexIterator> Iterators;
     TPriorityQueue<TScreenedPartIndexIterator*, TSmallVec<TScreenedPartIndexIterator*>, TIterKeyGreater> Heap;
@@ -186,7 +185,7 @@ class TKeyAccessSample {
     THashMap<TString, ui64> KeyRefCount;
 };
 
-void BuildStats(const TSubset& subset, TStats& stats, ui64 rowCountResolution, ui64 dataSizeResolution, const IPages* env);
+bool BuildStats(const TSubset& subset, TStats& stats, ui64 rowCountResolution, ui64 dataSizeResolution, IPages* env);
 void GetPartOwners(const TSubset& subset, THashSet<ui64>& partOwners);
 
 }}