Skip to content

Commit a7c35d9

Browse files
kungakunga
committed
Improve Shared Cache S3FIFO (#10825)
1 parent ba60050 commit a7c35d9

File tree

4 files changed

+215
-403
lines changed

4 files changed

+215
-403
lines changed

‎ydb/core/tablet_flat/shared_cache_s3fifo.h

+57-111
Original file line numberDiff line numberDiff line change
@@ -19,131 +19,64 @@ template <typename TPageTraits>
1919
class TS3FIFOGhostPageQueue {
2020
using TPageKey = typename TPageTraits::TPageKey;
2121

22-
struct TGhostPage {
23-
TPageKey Key;
24-
ui64 Size; // zero size is tombstone
25-
26-
TGhostPage(const TPageKey& key, ui64 size)
27-
: Key(key)
28-
, Size(size)
29-
{}
30-
};
31-
32-
struct TGhostPageHash {
33-
using is_transparent = void;
34-
35-
inline size_t operator()(const TGhostPage* ghost) const {
36-
return TPageTraits::GetHash(ghost->Key);
37-
}
38-
39-
inline size_t operator()(const TPageKey& key) const {
40-
return TPageTraits::GetHash(key);
41-
}
42-
};
43-
44-
struct TGhostPageEqual {
45-
using is_transparent = void;
46-
47-
inline bool operator()(const TGhostPage* left, const TGhostPage* right) const {
48-
return TPageTraits::Equals(left->Key, right->Key);
49-
}
50-
51-
inline bool operator()(const TGhostPage* left, const TPageKey& right) const {
52-
return TPageTraits::Equals(left->Key, right);
53-
}
54-
};
55-
5622
public:
57-
TS3FIFOGhostPageQueue(ui64 limit)
58-
: Limit(limit)
59-
{}
60-
61-
void Add(const TPageKey& key, ui64 size) {
62-
if (Y_UNLIKELY(size == 0)) {
63-
Y_DEBUG_ABORT_S("Empty " << TPageTraits::ToString(key) << " page");
64-
return;
65-
}
23+
bool Add(const TPageKey& key) {
24+
size_t hash = TPageTraits::GetHash(key);
6625

67-
TGhostPage* ghost = &GhostsQueue.emplace_back(key, size);
68-
if (Y_UNLIKELY(!GhostsSet.emplace(ghost).second)) {
69-
GhostsQueue.pop_back();
70-
Y_DEBUG_ABORT_S("Duplicated " << TPageTraits::ToString(key) << " page");
71-
return;
26+
if (GhostsSet.insert(hash).second) {
27+
GhostsQueue.push_back(hash);
28+
return true;
7229
}
73-
74-
Size += ghost->Size;
75-
76-
EvictWhileFull();
30+
31+
return false;
7732
}
7833

79-
bool Erase(const TPageKey& key, ui64 size) {
80-
if (auto it = GhostsSet.find(key); it != GhostsSet.end()) {
81-
TGhostPage* ghost = *it;
82-
Y_DEBUG_ABORT_UNLESS(ghost->Size == size);
83-
Y_ABORT_UNLESS(Size >= ghost->Size);
84-
Size -= ghost->Size;
85-
ghost->Size = 0; // mark as deleted
86-
GhostsSet.erase(it);
87-
return true;
34+
void Limit(size_t limit) {
35+
while (GhostsQueue.size() > limit) {
36+
bool erased = GhostsSet.erase(GhostsQueue.front());
37+
Y_DEBUG_ABORT_UNLESS(erased);
38+
GhostsQueue.pop_front();
8839
}
89-
return false;
9040
}
91-
92-
void UpdateLimit(ui64 limit) {
93-
Limit = limit;
94-
EvictWhileFull();
41+
42+
bool Contains(const TPageKey& key) {
43+
size_t hash = TPageTraits::GetHash(key);
44+
return GhostsSet.contains(hash);
9545
}
9646

9747
TString Dump() const {
9848
TStringBuilder result;
9949
size_t count = 0;
100-
ui64 size = 0;
101-
for (auto it = GhostsQueue.begin(); it != GhostsQueue.end(); it++) {
102-
const TGhostPage* ghost = &*it;
103-
if (ghost->Size) { // isn't deleted
104-
Y_DEBUG_ABORT_UNLESS(GhostsSet.contains(ghost));
105-
if (count != 0) result << ", ";
106-
result << "{" << TPageTraits::ToString(ghost->Key) << " " << ghost->Size << "b}";
107-
count++;
108-
size += ghost->Size;
109-
}
50+
for (size_t hash : GhostsQueue) {
51+
Y_DEBUG_ABORT_UNLESS(GhostsSet.contains(hash));
52+
if (count != 0) result << ", ";
53+
result << hash;
54+
count++;
11055
}
11156
Y_DEBUG_ABORT_UNLESS(GhostsSet.size() == count);
112-
Y_DEBUG_ABORT_UNLESS(Size == size);
11357
return result;
11458
}
11559

11660
private:
117-
void EvictWhileFull() {
118-
while (!GhostsQueue.empty() && Size > Limit) {
119-
TGhostPage* ghost = &GhostsQueue.front();
120-
if (ghost->Size) { // isn't deleted
121-
Y_ABORT_UNLESS(Size >= ghost->Size);
122-
Size -= ghost->Size;
123-
bool erased = GhostsSet.erase(ghost);
124-
Y_ABORT_UNLESS(erased);
125-
}
126-
GhostsQueue.pop_front();
127-
}
128-
}
129-
130-
ui64 Limit;
131-
ui64 Size = 0;
132-
// TODO: store ghost withing PageMap
133-
THashSet<TGhostPage*, TGhostPageHash, TGhostPageEqual> GhostsSet;
134-
TDeque<TGhostPage> GhostsQueue;
61+
// Note: only hashes are stored, all the collisions just ignored
62+
THashSet<size_t> GhostsSet;
63+
TDeque<size_t> GhostsQueue;
13564
};
13665

13766
template <typename TPage, typename TPageTraits>
13867
class TS3FIFOCache : public ICacheCache<TPage> {
13968
using TPageKey = typename TPageTraits::TPageKey;
14069

70+
static const ui32 MaxMainQueueReinserts = 20;
71+
14172
struct TLimit {
73+
ui64 TotalLimit;
14274
ui64 SmallQueueLimit;
14375
ui64 MainQueueLimit;
14476

14577
TLimit(ui64 limit)
146-
: SmallQueueLimit(limit / 10)
78+
: TotalLimit(limit)
79+
, SmallQueueLimit(limit / 10)
14780
, MainQueueLimit(limit - SmallQueueLimit)
14881
{}
14982
};
@@ -155,6 +88,7 @@ class TS3FIFOCache : public ICacheCache<TPage> {
15588

15689
ES3FIFOPageLocation Location;
15790
TIntrusiveList<TPage> Queue;
91+
ui64 Count = 0;
15892
ui64 Size = 0;
15993
};
16094

@@ -163,7 +97,6 @@ class TS3FIFOCache : public ICacheCache<TPage> {
16397
: Limit(limit)
16498
, SmallQueue(ES3FIFOPageLocation::SmallQueue)
16599
, MainQueue(ES3FIFOPageLocation::MainQueue)
166-
, GhostQueue(limit)
167100
{}
168101

169102
TIntrusiveList<TPage> EvictNext() override {
@@ -205,7 +138,6 @@ class TS3FIFOCache : public ICacheCache<TPage> {
205138
const ES3FIFOPageLocation location = TPageTraits::GetLocation(page);
206139
switch (location) {
207140
case ES3FIFOPageLocation::None:
208-
EraseGhost(page);
209141
break;
210142
case ES3FIFOPageLocation::SmallQueue:
211143
Erase(SmallQueue, page);
@@ -222,7 +154,6 @@ class TS3FIFOCache : public ICacheCache<TPage> {
222154

223155
void UpdateLimit(ui64 limit) override {
224156
Limit = limit;
225-
GhostQueue.UpdateLimit(limit);
226157
}
227158

228159
ui64 GetSize() const override {
@@ -242,6 +173,7 @@ class TS3FIFOCache : public ICacheCache<TPage> {
242173
count++;
243174
size += TPageTraits::GetSize(page);
244175
}
176+
Y_DEBUG_ABORT_UNLESS(queue.Count == count);
245177
Y_DEBUG_ABORT_UNLESS(queue.Size == size);
246178
};
247179

@@ -257,26 +189,33 @@ class TS3FIFOCache : public ICacheCache<TPage> {
257189

258190
private:
259191
TPage* EvictOneIfFull() {
260-
while (true) {
261-
if (!SmallQueue.Queue.Empty() && SmallQueue.Size > Limit.SmallQueueLimit) {
192+
ui32 mainQueueReinserts = 0;
193+
194+
while (GetSize() > Limit.TotalLimit) {
195+
if (SmallQueue.Size > Limit.SmallQueueLimit) {
262196
TPage* page = Pop(SmallQueue);
263197
if (ui32 frequency = TPageTraits::GetFrequency(page); frequency > 1) { // load inserts, first read touches, second read touches
198+
TPageTraits::SetFrequency(page, 0);
264199
Push(MainQueue, page);
265200
} else {
266-
if (frequency) TPageTraits::SetFrequency(page, 0);
201+
if (frequency) { // the page is used only once
202+
TPageTraits::SetFrequency(page, 0);
203+
}
267204
AddGhost(page);
268205
return page;
269206
}
270-
} else if (!MainQueue.Queue.Empty() && MainQueue.Size > Limit.MainQueueLimit) {
207+
} else {
271208
TPage* page = Pop(MainQueue);
272-
if (ui32 frequency = TPageTraits::GetFrequency(page); frequency > 0) {
209+
if (ui32 frequency = TPageTraits::GetFrequency(page); frequency > 0 && mainQueueReinserts < MaxMainQueueReinserts) {
210+
mainQueueReinserts++;
273211
TPageTraits::SetFrequency(page, frequency - 1);
274212
Push(MainQueue, page);
275213
} else {
214+
if (frequency) { // reinserts limit exceeded
215+
TPageTraits::SetFrequency(page, 0);
216+
}
276217
return page;
277218
}
278-
} else {
279-
break;
280219
}
281220
}
282221

@@ -295,7 +234,7 @@ class TS3FIFOCache : public ICacheCache<TPage> {
295234
TIntrusiveList<TPage> Insert(TPage* page) {
296235
Y_DEBUG_ABORT_UNLESS(TPageTraits::GetLocation(page) == ES3FIFOPageLocation::None);
297236

298-
Push(EraseGhost(page) ? MainQueue : SmallQueue, page);
237+
Push(IsGhost(page) ? MainQueue : SmallQueue, page);
299238
TPageTraits::SetFrequency(page, 0);
300239

301240
TIntrusiveList<TPage> evictedList;
@@ -307,11 +246,13 @@ class TS3FIFOCache : public ICacheCache<TPage> {
307246
}
308247

309248
TPage* Pop(TQueue& queue) {
310-
Y_DEBUG_ABORT_UNLESS(!queue.Queue.Empty());
249+
Y_ABORT_UNLESS(!queue.Queue.Empty());
311250
Y_ABORT_UNLESS(TPageTraits::GetLocation(queue.Queue.Front()) == queue.Location);
251+
Y_ABORT_UNLESS(queue.Count > 0);
312252
Y_ABORT_UNLESS(queue.Size >= TPageTraits::GetSize(queue.Queue.Front()));
313253

314254
TPage* page = queue.Queue.PopFront();
255+
queue.Count--;
315256
queue.Size -= TPageTraits::GetSize(page);
316257
TPageTraits::SetLocation(page, ES3FIFOPageLocation::None);
317258

@@ -322,25 +263,30 @@ class TS3FIFOCache : public ICacheCache<TPage> {
322263
Y_ABORT_UNLESS(TPageTraits::GetLocation(page) == ES3FIFOPageLocation::None);
323264

324265
queue.Queue.PushBack(page);
266+
queue.Count++;
325267
queue.Size += TPageTraits::GetSize(page);
326268
TPageTraits::SetLocation(page, queue.Location);
327269
}
328270

329271
void Erase(TQueue& queue, TPage* page) {
330272
Y_ABORT_UNLESS(TPageTraits::GetLocation(page) == queue.Location);
273+
Y_ABORT_UNLESS(queue.Count > 0);
331274
Y_ABORT_UNLESS(queue.Size >= TPageTraits::GetSize(page));
332275

333276
page->Unlink();
277+
queue.Count--;
334278
queue.Size -= TPageTraits::GetSize(page);
335279
TPageTraits::SetLocation(page, ES3FIFOPageLocation::None);
336280
}
337281

338282
void AddGhost(const TPage* page) {
339-
GhostQueue.Add(TPageTraits::GetKey(page), TPageTraits::GetSize(page));
283+
if (GhostQueue.Add(TPageTraits::GetKey(page))) {
284+
GhostQueue.Limit(SmallQueue.Count + MainQueue.Count);
285+
}
340286
}
341287

342-
bool EraseGhost(const TPage* page) {
343-
return GhostQueue.Erase(TPageTraits::GetKey(page), TPageTraits::GetSize(page));
288+
bool IsGhost(const TPage* page) {
289+
return GhostQueue.Contains(TPageTraits::GetKey(page));
344290
}
345291

346292
private:

0 commit comments

Comments
 (0)