Skip to content

Commit c73a73b

Browse files
serbel324marsaly79
serbel324
authored and
marsaly79
committed
Add BsCostTracker and advanced cost metrics, KIKIMR-17759 (ydb-platform#600)
* Add BsCostTracker * Remove redundant Y_UNUSDE * Remove redundant HugeBlobSize redefinition * Add estimation for huge writes
1 parent 11a6f64 commit c73a73b

8 files changed

+352
-2
lines changed

‎ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.cpp

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#include "blobstorage_cost_tracker.h"
2+
3+
namespace NKikimr {
4+
5+
class TBsCostModelMirror3dc : public TBsCostModelBase {};
6+
7+
class TBsCostModel4Plus2Block : public TBsCostModelBase {};
8+
9+
class TBsCostModelMirror3of4 : public TBsCostModelBase {};
10+
11+
TBsCostTracker::TBsCostTracker(const TBlobStorageGroupType& groupType, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters)
12+
: GroupType(groupType)
13+
, CostCounters(counters->GetSubgroup("subsystem", "advancedCost"))
14+
, UserDiskCost(CostCounters->GetCounter("UserDiskCost", true))
15+
, CompactionDiskCost(CostCounters->GetCounter("CompactionDiskCost", true))
16+
, ScrubDiskCost(CostCounters->GetCounter("ScrubDiskCost", true))
17+
, DefragDiskCost(CostCounters->GetCounter("DefragDiskCost", true))
18+
, InternalDiskCost(CostCounters->GetCounter("InternalDiskCost", true))
19+
{
20+
switch (GroupType.GetErasure()) {
21+
case TBlobStorageGroupType::ErasureMirror3dc:
22+
CostModel = std::make_unique<TBsCostModelMirror3dc>();
23+
break;
24+
case TBlobStorageGroupType::Erasure4Plus2Block:
25+
CostModel = std::make_unique<TBsCostModelMirror3dc>();
26+
break;
27+
case TBlobStorageGroupType::ErasureMirror3of4:
28+
CostModel = std::make_unique<TBsCostModelMirror3of4>();
29+
break;
30+
default:
31+
break;
32+
}
33+
}
34+
35+
} // NKikimr

‎ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h

+292
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
#pragma once
2+
3+
#include "defs.h"
4+
#include "vdisk_costmodel.h"
5+
#include "vdisk_events.h"
6+
#include "vdisk_handle_class.h"
7+
8+
#include <util/system/compiler.h>
9+
#include <ydb/core/blobstorage/base/blobstorage_events.h>
10+
11+
namespace NKikimr {
12+
13+
class TBsCostModelBase {
14+
public:
15+
virtual ~TBsCostModelBase() = default;
16+
17+
protected:
18+
// Disk Settings
19+
ui64 DeviceSeekTimeNs = 5'000'000;
20+
ui64 HugeBlobSize = 1'000'000; // depends on erasure
21+
22+
ui64 DeviceReadSpeedBps = 500 * 1'000'000; // 500 MB/sec
23+
ui64 DeviceReadBlockSize = 4 * 1'000; // 4 kB
24+
25+
ui64 DeviceWriteSpeedBps = 250 * 1'000'000; // 250 MB/sec
26+
ui64 DeviceWriteBlockSize = 4 * 1'000; // 4 kB
27+
ui64 PDiskWriteBlockSize = 4ull * 1'000'000; // 4MB
28+
29+
// Estimated Coefficients
30+
// cost = A + B * size
31+
double WriteA = 2520;
32+
double WriteB = 5.7668;
33+
34+
double ReadA = WriteA;
35+
double ReadB = WriteB;
36+
37+
double HugeWriteA = 1.26748409e+06;
38+
double HugeWriteB = 2.69514462e+01;
39+
40+
private:
41+
enum class EMemoryOperationType {
42+
Read = 0,
43+
Sync = 1,
44+
FullSync = 2
45+
};
46+
47+
static ui64 MemoryOperationCost(EMemoryOperationType t = EMemoryOperationType::Read) {
48+
static TVector<ui64> costs = { 1000, 5000, 100000 }; // Read, Sync, FullSync
49+
// NOTES: for Sync we don't know the cost, but we may assume that we read from memory
50+
return costs.at(size_t(t));
51+
}
52+
53+
public:
54+
void Update(const TCostModel& costModel) {
55+
DeviceSeekTimeNs = costModel.SeekTimeUs * 1000;
56+
DeviceReadSpeedBps = costModel.ReadSpeedBps;
57+
DeviceWriteSpeedBps = costModel.WriteSpeedBps;
58+
DeviceReadBlockSize = costModel.ReadBlockSize;
59+
DeviceWriteBlockSize = costModel.WriteBlockSize;
60+
HugeBlobSize = costModel.MinREALHugeBlobInBytes;
61+
}
62+
63+
protected:
64+
ui64 BatchedWriteCost(ui64 chunkSize) const {
65+
ui64 seekTime = 1. * chunkSize / PDiskWriteBlockSize * DeviceSeekTimeNs;
66+
ui64 writeTime = chunkSize * 1'000'000'000ull / DeviceWriteSpeedBps;
67+
return seekTime + writeTime;
68+
}
69+
70+
ui64 WriteCost(ui64 chunkSize) const {
71+
ui64 seekTime = 1. * chunkSize * DeviceSeekTimeNs;
72+
ui64 writeTime = chunkSize * 1'000'000'000ull / DeviceWriteSpeedBps;
73+
return seekTime + writeTime;
74+
}
75+
76+
ui64 HugeWriteCost(ui64 chunkSize) const {
77+
ui64 blocksNumber = (chunkSize + DeviceWriteBlockSize - 1) / DeviceWriteBlockSize;
78+
ui64 seekTime = 1. * blocksNumber * DeviceSeekTimeNs;
79+
ui64 writeTime = chunkSize * 1'000'000'000ull / DeviceWriteSpeedBps;
80+
return seekTime + writeTime;
81+
}
82+
83+
ui64 ReadCost(ui64 chunkSize) const {
84+
ui64 blocksNumber = (chunkSize + DeviceReadBlockSize - 1) / DeviceReadBlockSize;
85+
ui64 seekTime = 1. * blocksNumber * DeviceSeekTimeNs;
86+
ui64 readTime = chunkSize * 1'000'000'000ull / DeviceReadSpeedBps;
87+
return seekTime + readTime;
88+
}
89+
90+
ui64 EstimatedWriteCost(ui64 chunkSize) const {
91+
return WriteA + WriteB * chunkSize;
92+
}
93+
94+
ui64 EstimatedHugeWriteCost(ui64 chunkSize) const {
95+
return HugeWriteA + HugeWriteB * chunkSize;
96+
}
97+
98+
public:
99+
/////// DS Proxy requests
100+
/// READS
101+
virtual ui64 GetCost(const TEvBlobStorage::TEvGet&/* ev*/) const { return 0; }
102+
virtual ui64 GetCost(const TEvBlobStorage::TEvRange&/* ev*/) const { return 0; }
103+
104+
/// WRITES
105+
virtual ui64 GetCost(const TEvBlobStorage::TEvBlock&/* ev*/) const { return 0; }
106+
107+
virtual ui64 GetCost(const TEvBlobStorage::TEvCollectGarbage&/* ev*/) const { return 0; }
108+
109+
virtual ui64 GetCost(const TEvBlobStorage::TEvPut&/* ev*/) const { return 0; }
110+
111+
// PATCHES
112+
virtual ui64 GetCost(const TEvBlobStorage::TEvPatch&/* ev*/) const { return 0; }
113+
114+
/////// VDisk requests
115+
/// READS
116+
ui64 GetCost(const TEvBlobStorage::TEvVGet& ev) const {
117+
const auto &record = ev.Record;
118+
ui64 cost = 0;
119+
120+
// range query
121+
if (record.HasRangeQuery()) {
122+
if (record.GetIndexOnly()) {
123+
// in-memory only
124+
cost += MemoryOperationCost();
125+
} else {
126+
// we don't know cost of the query, it depends on number of elements and their size
127+
cost += DeviceSeekTimeNs + 2'000'000ull * 1'000'000'000 / DeviceReadSpeedBps;
128+
}
129+
}
130+
131+
// extreme queries
132+
for (const auto &x : record.GetExtremeQueries()) {
133+
ui64 size = 0;
134+
if (x.HasSize())
135+
size = x.GetSize();
136+
else {
137+
TLogoBlobID id(LogoBlobIDFromLogoBlobID(x.GetId()));
138+
size = id.BlobSize();
139+
}
140+
141+
cost += ReadCost(size);
142+
}
143+
144+
return cost;
145+
}
146+
147+
ui64 GetCost(const TEvBlobStorage::TEvVGetBlock&/* ev*/) const {
148+
return MemoryOperationCost();
149+
}
150+
151+
ui64 GetCost(const TEvBlobStorage::TEvVGetBarrier&/* ev*/) const {
152+
return MemoryOperationCost();
153+
}
154+
155+
/// WRITES
156+
ui64 GetCost(const TEvBlobStorage::TEvVBlock& ev) const {
157+
return EstimatedWriteCost(ev.GetCachedByteSize());
158+
}
159+
160+
ui64 GetCost(const TEvBlobStorage::TEvVCollectGarbage& ev) const {
161+
return EstimatedWriteCost(ev.GetCachedByteSize());
162+
}
163+
164+
ui64 GetCost(const TEvBlobStorage::TEvVPut& ev) const {
165+
const auto &record = ev.Record;
166+
const NKikimrBlobStorage::EPutHandleClass handleClass = record.GetHandleClass();
167+
const ui64 size = record.HasBuffer() ? record.GetBuffer().size() : ev.GetPayload(0).GetSize();
168+
169+
NPriPut::EHandleType handleType = NPriPut::HandleType(HugeBlobSize, handleClass, size);
170+
if (handleType == NPriPut::Log) {
171+
return EstimatedWriteCost(size);
172+
} else {
173+
return EstimatedHugeWriteCost(size);
174+
}
175+
}
176+
177+
ui64 GetCost(const TEvBlobStorage::TEvVMultiPut& ev) const {
178+
const auto &record = ev.Record;
179+
const NKikimrBlobStorage::EPutHandleClass handleClass = record.GetHandleClass();
180+
ui64 cost = 0;
181+
182+
for (ui64 idx = 0; idx < record.ItemsSize(); ++idx) {
183+
const ui64 size = ev.GetBufferBytes(idx);
184+
NPriPut::EHandleType handleType = NPriPut::HandleType(HugeBlobSize, handleClass, size);
185+
if (handleType == NPriPut::Log) {
186+
cost += EstimatedWriteCost(size);
187+
} else {
188+
cost += EstimatedHugeWriteCost(size);
189+
}
190+
}
191+
return cost;
192+
}
193+
194+
// PATCHES
195+
ui64 GetCost(const TEvBlobStorage::TEvVPatchStart&/* ev*/) const { return 0; }
196+
ui64 GetCost(const TEvBlobStorage::TEvVPatchDiff&/* ev*/) const { return 0; }
197+
ui64 GetCost(const TEvBlobStorage::TEvVPatchXorDiff&/* ev*/) const { return 0; }
198+
ui64 GetCost(const TEvBlobStorage::TEvVMovedPatch&/* ev*/) const { return 0; }
199+
200+
/////// PDisk requests
201+
// READS
202+
ui64 GetCost(const NPDisk::TEvChunkRead& ev) const {
203+
return ReadCost(ev.Size);
204+
}
205+
206+
// WRITES
207+
ui64 GetCost(const NPDisk::TEvChunkWrite& ev) const {
208+
if (ev.PriorityClass == NPriPut::Log) {
209+
return EstimatedWriteCost(ev.PartsPtr->Size());
210+
} else {
211+
return EstimatedHugeWriteCost(ev.PartsPtr->Size());
212+
}
213+
}
214+
};
215+
216+
class TBsCostModelMirror3dc;
217+
class TBsCostModel4Plus2Block;
218+
class TBsCostModelMirror3of4;
219+
220+
class TBsCostTracker {
221+
private:
222+
TBlobStorageGroupType GroupType;
223+
std::unique_ptr<TBsCostModelBase> CostModel;
224+
225+
const TIntrusivePtr<::NMonitoring::TDynamicCounters> CostCounters;
226+
227+
::NMonitoring::TDynamicCounters::TCounterPtr UserDiskCost;
228+
::NMonitoring::TDynamicCounters::TCounterPtr CompactionDiskCost;
229+
::NMonitoring::TDynamicCounters::TCounterPtr ScrubDiskCost;
230+
::NMonitoring::TDynamicCounters::TCounterPtr DefragDiskCost;
231+
::NMonitoring::TDynamicCounters::TCounterPtr InternalDiskCost;
232+
233+
public:
234+
TBsCostTracker(const TBlobStorageGroupType& groupType, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters);
235+
236+
template<class TEv>
237+
ui64 GetCost(const TEv& ev) const {
238+
ui64 cost = 0;
239+
if (CostModel) {
240+
cost = CostModel->GetCost(ev);
241+
} else {
242+
cost = 0;
243+
}
244+
return cost;
245+
}
246+
247+
/// SETTINGS
248+
void UpdateFromVDiskSettings(NKikimrBlobStorage::TVDiskCostSettings &settings) const;
249+
250+
public:
251+
void UpdateCostModel(const TCostModel& costModel) {
252+
if (CostModel) {
253+
CostModel->Update(costModel);
254+
}
255+
}
256+
257+
public:
258+
template<class TEvent>
259+
void CountUserRequest(const TEvent& ev) {
260+
*UserDiskCost += GetCost(ev);
261+
}
262+
263+
void CountUserCost(ui64 cost) {
264+
*UserDiskCost += cost;
265+
}
266+
267+
template<class TEvent>
268+
void CountCompactionRequest(const TEvent& ev) {
269+
*CompactionDiskCost += GetCost(ev);
270+
}
271+
272+
template<class TEvent>
273+
void CountScrubRequest(const TEvent& ev) {
274+
*UserDiskCost += GetCost(ev);
275+
}
276+
277+
template<class TEvent>
278+
void CountDefragRequest(const TEvent& ev) {
279+
*DefragDiskCost += GetCost(ev);
280+
}
281+
282+
template<class TEvent>
283+
void CountInternalRequest(const TEvent& ev) {
284+
*InternalDiskCost += GetCost(ev);
285+
}
286+
287+
void CountInternalCost(ui64 cost) {
288+
*InternalDiskCost += cost;
289+
}
290+
};
291+
292+
} // NKikimr

‎ydb/core/blobstorage/vdisk/common/vdisk_context.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ namespace NKikimr {
5757
, ReplPDiskWriteQuoter(std::move(replPDiskWriteQuoter))
5858
, ReplNodeRequestQuoter(std::move(replNodeRequestQuoter))
5959
, ReplNodeResponseQuoter(std::move(replNodeResponseQuoter))
60+
, CostTracker(std::make_shared<TBsCostTracker>(Top->GType, vdiskCounters))
6061
, OutOfSpaceState(Top->GetTotalVDisksNum(), Top->GetOrderNumber(ShortSelfVDisk))
6162
, CostMonGroup(vdiskCounters, "subsystem", "cost")
6263
, Logger(as ? ActorSystemLogger(as) : DevNullLogger())

‎ydb/core/blobstorage/vdisk/common/vdisk_context.h

+12
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#pragma once
2+
#include "blobstorage_cost_tracker.h"
23
#include "defs.h"
34
#include "memusage.h"
45
#include "vdisk_config.h"
@@ -67,6 +68,7 @@ namespace NKikimr {
6768
TString LocalRecoveryErrorStr;
6869

6970
std::unique_ptr<TCostModel> CostModel;
71+
std::shared_ptr<TBsCostTracker> CostTracker;
7072

7173
private:
7274
// Managing disk space
@@ -172,20 +174,30 @@ namespace NKikimr {
172174
if (CostModel) {
173175
CostMonGroup.DefragCostNs() += CostModel->GetCost(ev);
174176
}
177+
CostTracker->CountDefragRequest(ev);
175178
}
176179

177180
template<class TEvent>
178181
void CountScrubCost(const TEvent& ev) {
179182
if (CostModel) {
180183
CostMonGroup.ScrubCostNs() += CostModel->GetCost(ev);
181184
}
185+
CostTracker->CountScrubRequest(ev);
182186
}
183187

184188
template<class TEvent>
185189
void CountCompactionCost(const TEvent& ev) {
186190
if (CostModel) {
187191
CostMonGroup.CompactionCostNs() += CostModel->GetCost(ev);
188192
}
193+
CostTracker->CountCompactionRequest(ev);
194+
}
195+
196+
void UpdateCostModel(std::unique_ptr<TCostModel>&& newCostModel) {
197+
CostModel = std::move(newCostModel);
198+
if (CostModel) {
199+
CostTracker->UpdateCostModel(*CostModel);
200+
}
189201
}
190202

191203
private:

‎ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h

+4
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,16 @@ public:
5555
{
5656
COUNTER_INIT(LsmCompactionBytesRead, true);
5757
COUNTER_INIT(LsmCompactionBytesWritten, true);
58+
COUNTER_INIT(LsmCompactionReadRequests, true);
59+
COUNTER_INIT(LsmCompactionWriteRequests, true);
5860
COUNTER_INIT(LsmHugeBytesWritten, true);
5961
COUNTER_INIT(LsmLogBytesWritten, true);
6062
}
6163

6264
COUNTER_DEF(LsmCompactionBytesRead)
6365
COUNTER_DEF(LsmCompactionBytesWritten)
66+
COUNTER_DEF(LsmCompactionReadRequests)
67+
COUNTER_DEF(LsmCompactionWriteRequests)
6468
COUNTER_DEF(LsmHugeBytesWritten)
6569
COUNTER_DEF(LsmLogBytesWritten)
6670
};

0 commit comments

Comments
 (0)