Skip to content

Commit 0a93980

Browse files
kungakunga
authored
Merge 2df30eb into d6aa89c 
2 parents d6aa89c + 2df30eb commit 0a93980

13 files changed

+49
-208
lines changed

‎ydb/core/tablet_flat/flat_executor.cpp

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2362,7 +2362,7 @@ void TExecutor::CommitTransactionLog(TAutoPtr<TSeat> seat, TPageCollectionTxEnv
23622362
auto *bySwitchAux = aux.AddBySwitchAux();
23632363

23642364
TPageCollectionProtoHelper::Snap(snap, loaned->PartComponents, partSwitch.TableId, CompactionLogic->BorrowedPartLevel());
2365-
TPageCollectionProtoHelper(true, false).Do(bySwitchAux->AddHotBundles(), loaned->PartComponents);
2365+
TPageCollectionProtoHelper(true).Do(bySwitchAux->AddHotBundles(), loaned->PartComponents);
23662366

23672367
auto body = proto.SerializeAsString();
23682368
auto glob = CommitManager->Turns.One(commit->Refs, std::move(body), true);
@@ -2823,9 +2823,6 @@ void TExecutor::Handle(NSharedCache::TEvUpdated::TPtr &ev) {
28232823

28242824
for (auto &kv : msg->Actions) {
28252825
if (auto *info = PrivatePageCache->Info(kv.first)) {
2826-
for (auto &kvCorrected : kv.second.Accepted) {
2827-
PrivatePageCache->UpdateSharedBody(info, kvCorrected.first, std::move(kvCorrected.second));
2828-
}
28292826
for (ui32 pageId : kv.second.Dropped) {
28302827
PrivatePageCache->DropSharedBody(info, pageId);
28312828
}
@@ -3474,7 +3471,7 @@ void TExecutor::Handle(NOps::TEvResult *ops, TProdCompact *msg, bool cancelled)
34743471
const auto &newPart = result.Part;
34753472

34763473
TPageCollectionProtoHelper::Snap(snap, newPart, tableId, logicResult.Changes.NewPartsLevel);
3477-
TPageCollectionProtoHelper(true, false).Do(bySwitchAux->AddHotBundles(), newPart);
3474+
TPageCollectionProtoHelper(true).Do(bySwitchAux->AddHotBundles(), newPart);
34783475
}
34793476
}
34803477

@@ -3650,7 +3647,6 @@ void TExecutor::UpdateCounters(const TActorContext &ctx) {
36503647
Counters->Simple()[TExecutorCounters::CACHE_TOTAL_SHARED_BODY].Set(stats.TotalSharedBody);
36513648
Counters->Simple()[TExecutorCounters::CACHE_TOTAL_PINNED_BODY].Set(stats.TotalPinnedBody);
36523649
Counters->Simple()[TExecutorCounters::CACHE_TOTAL_EXCLUSIVE].Set(stats.TotalExclusive);
3653-
Counters->Simple()[TExecutorCounters::CACHE_TOTAL_SHARED_PENDING].Set(stats.TotalSharedPending);
36543650
Counters->Simple()[TExecutorCounters::CACHE_TOTAL_STICKY].Set(stats.TotalSticky);
36553651
}
36563652

@@ -3772,14 +3768,14 @@ TString TExecutor::BorrowSnapshot(ui32 table, const TTableSnapshotContext &snap,
37723768
for (const auto &partView : subset->Flatten) {
37733769
auto *x = proto.AddParts();
37743770

3775-
TPageCollectionProtoHelper(false, false).Do(x->MutableBundle(), partView);
3771+
TPageCollectionProtoHelper(false).Do(x->MutableBundle(), partView);
37763772
snap.Impl->Borrowed(Step(), table, partView->Label, loaner);
37773773
}
37783774

37793775
for (const auto &part : subset->ColdParts) {
37803776
auto *x = proto.AddParts();
37813777

3782-
TPageCollectionProtoHelper(false, false).Do(x->MutableBundle(), part);
3778+
TPageCollectionProtoHelper(false).Do(x->MutableBundle(), part);
37833779
snap.Impl->Borrowed(Step(), table, part->Label, loaner);
37843780
}
37853781

@@ -4172,7 +4168,6 @@ void TExecutor::RenderHtmlPage(NMon::TEvRemoteHttpInfo::TPtr &ev) const {
41724168
DIV_CLASS("row") {str << "Total bytes in shared cache: " << PrivatePageCache->GetStats().TotalSharedBody; }
41734169
DIV_CLASS("row") {str << "Total bytes in local cache: " << PrivatePageCache->GetStats().TotalPinnedBody; }
41744170
DIV_CLASS("row") {str << "Total bytes exclusive to local cache: " << PrivatePageCache->GetStats().TotalExclusive; }
4175-
DIV_CLASS("row") {str << "Total bytes in transit to shared cache: " << PrivatePageCache->GetStats().TotalSharedPending; }
41764171
DIV_CLASS("row") {str << "Total bytes marked as sticky: " << PrivatePageCache->GetStats().TotalSticky; }
41774172

41784173
if (GcLogic) {

‎ydb/core/tablet_flat/flat_executor.proto

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,6 @@ message TLargeGlobId {
1515
}
1616

1717
message TPageCollection {
18-
message TPage {
19-
required uint32 Id = 1;
20-
required bytes Body = 2;
21-
}
22-
2318
repeated NKikimrProto.TLogoBlobID MetaId = 1; // Replaced by TLargeGlobId
2419

2520
optional TLargeGlobId LargeGlobId = 6;
@@ -30,7 +25,7 @@ message TPageCollection {
3025
// packed page collection is called HotBundle.
3126

3227
optional bytes Meta = 2;
33-
repeated TPage Pages = 5;
28+
reserved 5; // Pages
3429
}
3530

3631
message TBundle {

‎ydb/core/tablet_flat/flat_executor_tx_env.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,11 @@ namespace NTabletFlatExecutor {
3838
return { !ReadMissingReferences, page };
3939
}
4040

41-
const TSharedData* TryGetPage(const TPart* part, TPageId page, TGroupId groupId) override
41+
const TSharedData* TryGetPage(const TPart* part, TPageId pageId, TGroupId groupId) override
4242
{
4343
auto *partStore = CheckedCast<const NTable::TPartStore*>(part);
4444

45-
return Lookup(partStore->PageCollections.at(groupId.Index).Get(), page);
45+
return Lookup(partStore->PageCollections.at(groupId.Index).Get(), pageId);
4646
}
4747

4848
void EnableReadMissingReferences() noexcept {

‎ydb/core/tablet_flat/flat_part_loader.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ namespace NTable {
8989
if (type != EPage::FlatIndex) {
9090
// hack: saving flat index to private cache will break sticky logic
9191
// keep it in shared cache only for now
92-
Cache->Fill(std::move(loaded), NeedIn(type));
92+
Cache->Fill(loaded.PageId, std::move(loaded.Page), NeedIn(type));
9393
}
9494
}
9595
}

‎ydb/core/tablet_flat/flat_part_outset.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
#include "defs.h"
44
#include "flat_sausage_packet.h"
5-
#include "flat_sausage_fetch.h"
65

76
namespace NKikimr {
87
namespace NTable {
@@ -12,7 +11,6 @@ namespace NTable {
1211
NPageCollection::TLargeGlobId LargeGlobId;
1312
// loaded meta page
1413
TIntrusiveConstPtr<NPageCollection::TPageCollection> Packet;
15-
TVector<NPageCollection::TLoadedPage> Sticky;
1614

1715
void ParsePacket(TSharedData meta);
1816
};

‎ydb/core/tablet_flat/flat_part_store.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -154,9 +154,6 @@ class TPartStore : public TPart, public IBundle {
154154

155155
for (auto &one: components) {
156156
caches.emplace_back(new TCache(std::move(one.Packet)));
157-
158-
for (auto &page: one.Sticky)
159-
caches.back()->Fill(page, true);
160157
}
161158

162159
return caches;

‎ydb/core/tablet_flat/flat_sausagecache.cpp

Lines changed: 7 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ namespace NTabletFlatExecutor {
77
TPrivatePageCache::TPage::TPage(size_t size, TPageId pageId, TInfo* info)
88
: LoadState(LoadStateNo)
99
, Sticky(false)
10-
, SharedPending(false)
1110
, Id(pageId)
1211
, Size(size)
1312
, Info(info)
@@ -47,20 +46,15 @@ void TPrivatePageCache::RegisterPageCollection(TIntrusivePtr<TInfo> info) {
4746

4847
for (const auto& kv : info->PageMap) {
4948
auto* page = kv.second.Get();
50-
Y_DEBUG_ABORT_UNLESS(page);
51-
52-
if (page->SharedBody)
53-
Stats.TotalSharedBody += page->Size;
49+
Y_ABORT_UNLESS(page);
50+
Y_ABORT_UNLESS(page->SharedBody, "New filled pages can't be without a shared body");
51+
52+
Stats.TotalSharedBody += page->Size;
5453
if (page->PinnedBody)
5554
Stats.TotalPinnedBody += page->Size;
56-
if (page->PinnedBody && !page->SharedBody)
57-
Stats.TotalExclusive += page->Size;
5855
if (page->Sticky)
5956
Stats.TotalSticky += page->Size;
6057

61-
Y_DEBUG_ABORT_UNLESS(!page->SharedPending, "New page shouldn't be shared pending");
62-
TryShareBody(page);
63-
6458
TryUnload(page);
6559
Y_DEBUG_ABORT_UNLESS(!page->IsUnnecessary());
6660
}
@@ -132,8 +126,6 @@ bool TPrivatePageCache::UnlockPageCollection(TLogoBlobID id) {
132126
Stats.TotalPinnedBody -= page->Size;
133127
if (page->PinnedBody && !page->SharedBody)
134128
Stats.TotalExclusive -= page->Size;
135-
if (page->SharedPending)
136-
Stats.TotalSharedPending -= page->Size;
137129
if (page->Sticky)
138130
Stats.TotalSticky -= page->Size;
139131
}
@@ -204,8 +196,6 @@ std::pair<ui32, ui64> TPrivatePageCache::Request(TVector<ui32> &pages, TPrivateP
204196
TPage *page = info->EnsurePage(*it);
205197
switch (page->LoadState) {
206198
case TPage::LoadStateNo:
207-
Y_DEBUG_ABORT_UNLESS(!page->SharedPending, "Trying to load a page that may be restored");
208-
[[fallthrough]];
209199
case TPage::LoadStateRequestedAsync:
210200
page->LoadState = TPage::LoadStateRequested;
211201
bytesToRequest += page->Size;
@@ -261,8 +251,8 @@ void TPrivatePageCache::TryLoad(TPage *page) {
261251

262252
void TPrivatePageCache::TPrivatePageCache::TryUnload(TPage *page) {
263253
if (page->LoadState == TPage::LoadStateLoaded) {
264-
if (!page->SharedPending && !page->PinPad && !page->Sticky) {
265-
ToTouchShared[page->Info->Id][page->Id];
254+
if (!page->PinPad && !page->Sticky) {
255+
ToTouchShared[page->Info->Id].insert(page->Id);
266256
page->LoadState = TPage::LoadStateNo;
267257
if (Y_LIKELY(page->PinnedBody)) {
268258
Stats.TotalPinnedBody -= page->Size;
@@ -291,19 +281,6 @@ void TPrivatePageCache::TPrivatePageCache::TryEraseIfUnnecessary(TPage *page) {
291281
}
292282
}
293283

294-
void TPrivatePageCache::TPrivatePageCache::TryShareBody(TPage *page) {
295-
if (page->LoadState == TPage::LoadStateLoaded) {
296-
auto &x = ToTouchShared[page->Info->Id][page->Id];
297-
if (!page->SharedPending && !page->SharedBody && page->PinnedBody) {
298-
// We keep pinned body around until it's either
299-
// accepted or dropped by the shared cache
300-
page->SharedPending = true;
301-
Stats.TotalSharedPending += page->Size;
302-
x = page->PinnedBody;
303-
}
304-
}
305-
}
306-
307284
const TSharedData* TPrivatePageCache::Lookup(TPageId pageId, TInfo *info) {
308285
TPage *page = info->EnsurePage(pageId);
309286

@@ -502,45 +479,11 @@ void TPrivatePageCache::ResetTouchesAndToLoad(bool verifyEmpty) {
502479
Stats.CurrentCacheMisses = 0;
503480
}
504481

505-
void TPrivatePageCache::UpdateSharedBody(TInfo *info, TPageId pageId, TSharedPageRef shared) {
506-
TPage *page = info->GetPage(pageId);
507-
if (!page)
508-
return;
509-
510-
// Note: shared cache may accept a pending page if it is used by multiple private caches
511-
// (for example, used by tablet and its follower)
512-
if (Y_UNLIKELY(!page->SharedPending)) {
513-
return;
514-
}
515-
Y_DEBUG_ABORT_UNLESS(page->LoadState == TPage::LoadStateLoaded, "Shared pending page should be loaded");
516-
517-
// Shared cache accepted our page and provided its shared reference
518-
Stats.TotalSharedPending -= page->Size;
519-
page->SharedPending = false;
520-
if (Y_LIKELY(!page->SharedBody)) {
521-
Stats.TotalSharedBody += page->Size;
522-
if (Y_LIKELY(page->PinnedBody)) {
523-
Stats.TotalExclusive -= page->Size;
524-
}
525-
}
526-
page->SharedBody = std::move(shared);
527-
TryUnload(page);
528-
}
529-
530482
void TPrivatePageCache::DropSharedBody(TInfo *info, TPageId pageId) {
531483
TPage *page = info->GetPage(pageId);
532484
if (!page)
533485
return;
534486

535-
// Note: shared cache may drop a pending page if it is used by multiple private caches
536-
// (for example, used by tablet and its follower)
537-
if (Y_UNLIKELY(page->SharedPending)) {
538-
// Shared cache rejected our page so we should drop it too
539-
Stats.TotalSharedPending -= page->Size;
540-
page->SharedPending = false;
541-
TryUnload(page);
542-
}
543-
544487
if (!page->SharedBody.IsUsed()) {
545488
if (Y_LIKELY(page->SharedBody)) {
546489
Stats.TotalSharedBody -= page->Size;
@@ -568,8 +511,6 @@ TPrivatePageCache::TPage::TWaitQueuePtr TPrivatePageCache::ProvideBlock(
568511
Stats.TotalPinnedBody -= page->Size;
569512
if (Y_UNLIKELY(page->PinnedBody && !page->SharedBody))
570513
Stats.TotalExclusive -= page->Size;
571-
if (Y_UNLIKELY(page->SharedPending))
572-
Stats.TotalSharedPending -= page->Size;
573514

574515
// Note: we must be careful not to accidentally drop the sticky bit
575516
page->Fill(std::move(loaded.Page), page->Sticky);
@@ -603,7 +544,7 @@ THashMap<TLogoBlobID, TIntrusivePtr<TPrivatePageCache::TInfo>> TPrivatePageCache
603544
return ret;
604545
}
605546

606-
THashMap<TLogoBlobID, THashMap<TPrivatePageCache::TPageId, TSharedData>> TPrivatePageCache::GetPrepareSharedTouched() {
547+
THashMap<TLogoBlobID, THashSet<TPrivatePageCache::TPageId>> TPrivatePageCache::GetPrepareSharedTouched() {
607548
return std::move(ToTouchShared);
608549
}
609550

‎ydb/core/tablet_flat/flat_sausagecache.h

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ class TPrivatePageCache {
3232
ui64 TotalSharedBody = 0; // total number of bytes currently referenced from shared cache
3333
ui64 TotalPinnedBody = 0; // total number of bytes currently pinned in memory
3434
ui64 TotalExclusive = 0; // total number of bytes exclusive to this cache (not from shared cache)
35-
ui64 TotalSharedPending = 0; // total number of bytes waiting for transfer to shared cache
3635
ui64 TotalSticky = 0; // total number of bytes marked as sticky (never unloaded from memory)
3736
ui64 PinnedSetSize = 0; // number of bytes pinned by transactions (even those not currently loaded)
3837
ui64 PinnedLoadSize = 0; // number of bytes pinned by transactions (which are currently being loaded)
@@ -54,7 +53,6 @@ class TPrivatePageCache {
5453

5554
ui32 LoadState : 2;
5655
ui32 Sticky : 1;
57-
ui32 SharedPending : 1;
5856

5957
const TPageId Id;
6058
const size_t Size;
@@ -74,22 +72,13 @@ class TPrivatePageCache {
7472
return (
7573
LoadState == LoadStateNo &&
7674
!Sticky &&
77-
!SharedPending &&
7875
!PinPad &&
7976
!WaitQueue &&
8077
!SharedBody);
8178
}
8279

83-
void Fill(TSharedData data, bool sticky = false) {
84-
Y_DEBUG_ABORT_UNLESS(!SharedBody && !SharedPending, "Populating cache with shared data already present");
80+
void Fill(TSharedPageRef shared, bool sticky) {
8581
Sticky = sticky;
86-
LoadState = LoadStateLoaded;
87-
PinnedBody = std::move(data);
88-
}
89-
90-
void Fill(TSharedPageRef shared, bool sticky = false) {
91-
Sticky = sticky;
92-
SharedPending = false;
9382
SharedBody = std::move(shared);
9483
LoadState = LoadStateLoaded;
9584
PinnedBody = TPinnedPageRef(SharedBody).GetData();
@@ -130,14 +119,6 @@ class TPrivatePageCache {
130119
return page;
131120
}
132121

133-
void Fill(const NPageCollection::TLoadedPage &paged, bool sticky = false) noexcept {
134-
EnsurePage(paged.PageId)->Fill(paged.Data, sticky);
135-
}
136-
137-
void Fill(NSharedCache::TEvResult::TLoaded&& loaded, bool sticky = false) noexcept {
138-
EnsurePage(loaded.PageId)->Fill(std::move(loaded.Page), sticky);
139-
}
140-
141122
void Fill(ui32 pageId, TSharedPageRef page, bool sticky) noexcept {
142123
EnsurePage(pageId)->Fill(std::move(page), sticky);
143124
}
@@ -178,16 +159,15 @@ class TPrivatePageCache {
178159
THashMap<TPrivatePageCache::TInfo*, TVector<ui32>> GetToLoad() const;
179160
void ResetTouchesAndToLoad(bool verifyEmpty);
180161

181-
void UpdateSharedBody(TInfo *collectionInfo, TPageId pageId, TSharedPageRef shared);
182162
void DropSharedBody(TInfo *collectionInfo, TPageId pageId);
183163

184164
TPage::TWaitQueuePtr ProvideBlock(NSharedCache::TEvResult::TLoaded&& loaded, TInfo *collectionInfo);
185165
THashMap<TLogoBlobID, TIntrusivePtr<TInfo>> DetachPrivatePageCache();
186-
THashMap<TLogoBlobID, THashMap<TPageId, TSharedData>> GetPrepareSharedTouched();
166+
THashMap<TLogoBlobID, THashSet<TPageId>> GetPrepareSharedTouched();
187167

188168
private:
189169
THashMap<TLogoBlobID, TIntrusivePtr<TInfo>> PageCollections;
190-
THashMap<TLogoBlobID, THashMap<TPageId, TSharedData>> ToTouchShared;
170+
THashMap<TLogoBlobID, THashSet<TPageId>> ToTouchShared;
191171

192172
TStats Stats;
193173

@@ -200,7 +180,6 @@ class TPrivatePageCache {
200180
void TryLoad(TPage *page);
201181
void TryUnload(TPage *page);
202182
void TryEraseIfUnnecessary(TPage *page);
203-
void TryShareBody(TPage *page);
204183
};
205184

206185
}}

0 commit comments

Comments
 (0)