Skip to content

Commit 2a2fa08

Browse files
adameatadameat
authored
Merge 647cc81 into 301ab14 
2 parents 301ab14 + 647cc81 commit 2a2fa08

17 files changed

+319
-127
lines changed

‎ydb/core/base/pool_stats_collector.cpp

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,24 @@ class TStatsCollectingActor : public NActors::TStatsCollectingActor {
4646
void OnWakeup(const TActorContext &ctx) override {
4747
MiniKQLPoolStats.Update();
4848

49-
TVector<std::tuple<TString, double, ui32, ui32>> pools;
49+
auto systemUpdate = std::make_unique<NNodeWhiteboard::TEvWhiteboard::TEvSystemStateUpdate>();
50+
ui32 coresTotal = 0;
51+
double coresUsed = 0;
5052
for (const auto& pool : PoolCounters) {
51-
pools.emplace_back(pool.Name, pool.Usage, pool.Threads, pool.LimitThreads);
53+
auto& pb = *systemUpdate->Record.AddPoolStats();
54+
pb.SetName(pool.Name);
55+
pb.SetUsage(pool.Usage);
56+
pb.SetThreads(static_cast<ui32>(pool.Threads));
57+
pb.SetLimit(static_cast<ui32>(pool.LimitThreads));
58+
if (pool.Name != "IO") {
59+
coresTotal += static_cast<ui32>(pool.DefaultThreads);
60+
}
61+
coresUsed += pool.Usage * pool.LimitThreads;
5262
}
63+
systemUpdate->Record.SetCoresTotal(coresTotal);
64+
systemUpdate->Record.SetCoresUsed(coresUsed);
5365

54-
ctx.Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(ctx.SelfID.NodeId()), new NNodeWhiteboard::TEvWhiteboard::TEvSystemStateUpdate(pools));
66+
ctx.Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(ctx.SelfID.NodeId()), systemUpdate.release());
5567
}
5668

5769
private:

‎ydb/core/node_whiteboard/node_whiteboard.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,5 +534,8 @@ struct WhiteboardResponse<TEvWhiteboard::TEvNodeStateRequest> {
534534
using Type = TEvWhiteboard::TEvNodeStateResponse;
535535
};
536536

537+
template<typename TResponseType>
538+
::google::protobuf::RepeatedField<int> GetDefaultWhiteboardFields();
539+
537540
} // NNodeWhiteboard
538541
} // NKikimr

‎ydb/core/protos/node_whiteboard.proto

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,7 @@ message TSystemStateInfo {
319319
optional string DataCenter = 16;
320320
optional uint32 RackId = 17;
321321
optional string Rack = 18;
322-
optional string Host = 19;
322+
optional string Host = 19 [(DefaultField) = true];
323323
optional string Version = 20 [(DefaultField) = true];
324324
repeated TPoolStats PoolStats = 21 [(DefaultField) = true];
325325
repeated TEndpoint Endpoints = 22 [(DefaultField) = true];
@@ -338,6 +338,8 @@ message TSystemStateInfo {
338338
optional TNodeSharedCache SharedCacheStats = 35; // TODO: use memory stats
339339
optional uint32 TotalSessions = 36 [(DefaultField) = true];
340340
optional string NodeName = 37 [(DefaultField) = true];
341+
optional double CoresUsed = 39;
342+
optional uint32 CoresTotal = 40;
341343
}
342344

343345
message TEvSystemStateRequest {

‎ydb/core/tablet/node_whiteboard.cpp

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,7 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
477477
}
478478
}
479479

480-
static void SelectiveCopy(::google::protobuf::Message& protoTo, const ::google::protobuf::Message& protoFrom, const ::google::protobuf::RepeatedField<arc_i32>& fields) {
480+
static void SelectiveCopy(::google::protobuf::Message& protoTo, const ::google::protobuf::Message& protoFrom, const ::google::protobuf::RepeatedField<int>& fields) {
481481
using namespace ::google::protobuf;
482482
const Descriptor& descriptor = *protoTo.GetDescriptor();
483483
const Reflection& reflectionTo = *protoTo.GetReflection();
@@ -490,24 +490,6 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
490490
}
491491
}
492492

493-
template<typename TMessage>
494-
static ::google::protobuf::RepeatedField<arc_i32> GetDefaultFields(const TMessage& message) {
495-
using namespace ::google::protobuf;
496-
const Descriptor& descriptor = *message.GetDescriptor();
497-
::google::protobuf::RepeatedField<arc_i32> defaultFields;
498-
int fieldCount = descriptor.field_count();
499-
for (int index = 0; index < fieldCount; ++index) {
500-
const FieldDescriptor* field = descriptor.field(index);
501-
const auto& options(field->options());
502-
if (options.HasExtension(NKikimrWhiteboard::DefaultField)) {
503-
if (options.GetExtension(NKikimrWhiteboard::DefaultField)) {
504-
defaultFields.Add(field->number());
505-
}
506-
}
507-
}
508-
return defaultFields;
509-
}
510-
511493
template<typename TMessage, typename TRequest>
512494
static void Copy(TMessage& to, const TMessage& from, const TRequest& request) {
513495
if (request.FieldsRequiredSize() > 0) {
@@ -517,8 +499,7 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
517499
SelectiveCopy(to, from, request.GetFieldsRequired());
518500
}
519501
} else {
520-
static auto defaultFields = GetDefaultFields(to);
521-
SelectiveCopy(to, from, defaultFields);
502+
SelectiveCopy(to, from, GetDefaultWhiteboardFields<TMessage>());
522503
}
523504
}
524505

@@ -1121,6 +1102,30 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
11211102
}
11221103
};
11231104

1105+
template<typename TMessage>
1106+
::google::protobuf::RepeatedField<int> InitDefaultWhiteboardFields() {
1107+
using namespace ::google::protobuf;
1108+
const Descriptor& descriptor = *TMessage::GetDescriptor();
1109+
::google::protobuf::RepeatedField<int> defaultFields;
1110+
int fieldCount = descriptor.field_count();
1111+
for (int index = 0; index < fieldCount; ++index) {
1112+
const FieldDescriptor* field = descriptor.field(index);
1113+
const auto& options(field->options());
1114+
if (options.HasExtension(NKikimrWhiteboard::DefaultField)) {
1115+
if (options.GetExtension(NKikimrWhiteboard::DefaultField)) {
1116+
defaultFields.Add(field->number());
1117+
}
1118+
}
1119+
}
1120+
return defaultFields;
1121+
}
1122+
1123+
template<typename TMessage>
1124+
::google::protobuf::RepeatedField<int> GetDefaultWhiteboardFields() {
1125+
static ::google::protobuf::RepeatedField<int> defaultFields = InitDefaultWhiteboardFields<TMessage>();
1126+
return defaultFields;
1127+
}
1128+
11241129
IActor* CreateNodeWhiteboardService() {
11251130
return new TNodeWhiteboardService();
11261131
}

‎ydb/core/viewer/json_handlers_viewer.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ void InitViewerStorageUsageJsonHandler(TJsonHandlers &handlers) {
203203
}
204204

205205
void InitViewerClusterJsonHandler(TJsonHandlers& handlers) {
206-
handlers.AddHandler("/viewer/cluster", new TJsonHandler<TJsonCluster>(TJsonCluster::GetSwagger()), 4);
206+
handlers.AddHandler("/viewer/cluster", new TJsonHandler<TJsonCluster>(TJsonCluster::GetSwagger()), 5);
207207
}
208208

209209
void InitViewerLabeledCountersJsonHandler(TJsonHandlers &handlers) {
@@ -219,15 +219,15 @@ void InitViewerHiveStatsJsonHandler(TJsonHandlers& handlers) {
219219
}
220220

221221
void InitViewerTenantInfoJsonHandler(TJsonHandlers &handlers) {
222-
handlers.AddHandler("/viewer/tenantinfo", new TJsonHandler<TJsonTenantInfo>(TJsonTenantInfo::GetSwagger()), 2);
222+
handlers.AddHandler("/viewer/tenantinfo", new TJsonHandler<TJsonTenantInfo>(TJsonTenantInfo::GetSwagger()), 3);
223223
}
224224

225225
void InitViewerWhoAmIJsonHandler(TJsonHandlers& handlers) {
226226
handlers.AddHandler("/viewer/whoami", new TJsonHandler<TJsonWhoAmI>(TJsonWhoAmI::GetSwagger()));
227227
}
228228

229229
void InitViewerQueryJsonHandler(TJsonHandlers& handlers) {
230-
handlers.AddHandler("/viewer/query", new TJsonHandler<TJsonQuery>(TJsonQuery::GetSwagger()), 3);
230+
handlers.AddHandler("/viewer/query", new TJsonHandler<TJsonQuery>(TJsonQuery::GetSwagger()), 4);
231231
}
232232

233233
void InitViewerNetInfoJsonHandler(TJsonHandlers& handlers) {
@@ -243,7 +243,7 @@ void InitViewerHealthCheckJsonHandler(TJsonHandlers& handlers) {
243243
}
244244

245245
void InitViewerNodesJsonHandler(TJsonHandlers& handlers) {
246-
handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 9);
246+
handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 11);
247247
}
248248

249249
void InitViewerACLJsonHandler(TJsonHandlers &jsonHandlers) {

‎ydb/core/viewer/json_local_rpc.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,10 +128,9 @@ class TJsonLocalRpc : public TViewerPipeClient {
128128
ReplyAndPassAway(GetHTTPBADREQUEST("text/plain", e.what()));
129129
return false;
130130
}
131-
} else {
132-
const auto& params(Event->Get()->Request.GetParams());
133-
Params2Proto(params, request);
134131
}
132+
const auto& params(Event->Get()->Request.GetParams());
133+
Params2Proto(params, request);
135134
if (!ValidateRequest(request)) {
136135
return false;
137136
}

‎ydb/core/viewer/json_pipe_req.cpp

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "json_pipe_req.h"
2+
#include <library/cpp/json/json_reader.h>
23
#include <library/cpp/json/json_writer.h>
34

45
namespace NKikimr::NViewer {
@@ -25,7 +26,28 @@ TViewerPipeClient::TViewerPipeClient(IViewer* viewer, NMon::TEvHttpInfo::TPtr& e
2526
: Viewer(viewer)
2627
, Event(ev)
2728
{
28-
InitConfig(Event->Get()->Request.GetParams());
29+
TCgiParameters params = Event->Get()->Request.GetParams();
30+
if (Event->Get()->Request.GetHeader("Content-Type") == "application/json") {
31+
NJson::TJsonValue jsonData;
32+
if (NJson::ReadJsonTree(Event->Get()->Request.GetPostContent(), &jsonData)) {
33+
if (jsonData.IsMap()) {
34+
for (const auto& [key, value] : jsonData.GetMap()) {
35+
switch (value.GetType()) {
36+
case NJson::EJsonValueType::JSON_STRING:
37+
case NJson::EJsonValueType::JSON_INTEGER:
38+
case NJson::EJsonValueType::JSON_UINTEGER:
39+
case NJson::EJsonValueType::JSON_DOUBLE:
40+
case NJson::EJsonValueType::JSON_BOOLEAN:
41+
params.InsertUnescaped(key, value.GetStringRobust());
42+
break;
43+
default:
44+
break;
45+
}
46+
}
47+
}
48+
}
49+
}
50+
InitConfig(params);
2951
NWilson::TTraceId traceId;
3052
TStringBuf traceparent = Event->Get()->Request.GetHeader("traceparent");
3153
if (traceparent) {
@@ -645,10 +667,20 @@ TRequestState TViewerPipeClient::GetRequest() const {
645667
}
646668

647669
void TViewerPipeClient::ReplyAndPassAway(TString data, const TString& error) {
670+
TString message = error;
648671
Send(Event->Sender, new NMon::TEvHttpInfoRes(data, 0, NMon::IEvHttpInfoRes::EContentType::Custom));
672+
if (message.empty()) {
673+
TStringBuf dataParser(data);
674+
if (dataParser.NextTok(' ') == "HTTP/1.1") {
675+
TStringBuf code = dataParser.NextTok(' ');
676+
if (code.size() == 3 && code[0] != '2') {
677+
message = dataParser.NextTok('\n');
678+
}
679+
}
680+
}
649681
if (Span) {
650-
if (error) {
651-
Span.EndError(error);
682+
if (message) {
683+
Span.EndError(message);
652684
} else {
653685
Span.EndOk();
654686
}

‎ydb/core/viewer/protos/viewer.proto

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,7 @@ message TClusterInfo {
322322
uint32 NodesTotal = 10;
323323
uint32 NodesAlive = 11;
324324
uint32 NumberOfCpus = 20;
325+
uint32 CoresTotal = 24;
325326
double CoresUsed = 21;
326327
double LoadAverage = 22;
327328
repeated NKikimrWhiteboard.TSystemStateInfo.TPoolStats PoolStats = 23;
@@ -358,7 +359,7 @@ message TStorageUsage {
358359
SSD = 2;
359360
}
360361
EType Type = 1;
361-
uint64 Size = 2;
362+
optional uint64 Size = 2;
362363
uint64 Limit = 3;
363364
uint64 SoftQuota = 4;
364365
uint64 HardQuota = 5;
@@ -394,6 +395,7 @@ message TTenant {
394395
Ydb.Cms.DatabaseQuotas DatabaseQuotas = 42;
395396
repeated TStorageUsage TablesStorage = 44;
396397
repeated TStorageUsage DatabaseStorage = 45;
398+
uint32 CoresTotal = 50;
397399
}
398400

399401
message TTenants {

‎ydb/core/viewer/storage_groups.h

Lines changed: 27 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,6 @@ class TStorageGroups : public TViewerPipeClient {
151151
std::unordered_set<TGroupId> FilterGroupIds;
152152
std::unordered_set<TNodeId> FilterNodeIds;
153153
std::unordered_set<ui32> FilterPDiskIds;
154-
std::vector<TNodeId> SubscriptionNodeIds;
155154

156155
enum class EWith {
157156
Everything,
@@ -894,22 +893,6 @@ class TStorageGroups : public TViewerPipeClient {
894893
Schedule(TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup(TimeoutFinal)); // timeout for the rest
895894
}
896895

897-
void PassAway() override {
898-
std::vector<bool> passedNodes;
899-
for (const TNodeId nodeId : SubscriptionNodeIds) {
900-
if (passedNodes.size() <= nodeId) {
901-
passedNodes.resize(nodeId + 1);
902-
} else {
903-
if (passedNodes[nodeId]) {
904-
continue;
905-
}
906-
}
907-
Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe());
908-
passedNodes[nodeId] = true;
909-
}
910-
TBase::PassAway();
911-
}
912-
913896
void ApplyFilter() {
914897
// database pre-filter, affects TotalGroups count
915898
if (!DatabaseStoragePools.empty()) {
@@ -1042,20 +1025,29 @@ class TStorageGroups : public TViewerPipeClient {
10421025
With = EWith::Everything;
10431026
GroupsByGroupId.clear();
10441027
}
1045-
if (!Filter.empty() && FieldsAvailable.test(+EGroupFields::PoolName) && FieldsAvailable.test(+EGroupFields::GroupId)) {
1046-
TVector<TString> filterWords = SplitString(Filter, " ");
1047-
TGroupView groupView;
1048-
for (TGroup* group : GroupView) {
1049-
for (const TString& word : filterWords) {
1050-
if (group->PoolName.Contains(word) || ::ToString(group->GroupId).Contains(word)) {
1051-
groupView.push_back(group);
1052-
break;
1028+
if (!Filter.empty()) {
1029+
bool allFieldsPresent =
1030+
(!FieldsRequired.test(+EGroupFields::GroupId) || FieldsAvailable.test(+EGroupFields::GroupId)) &&
1031+
(!FieldsRequired.test(+EGroupFields::PoolName) || FieldsAvailable.test(+EGroupFields::PoolName));
1032+
if (allFieldsPresent) {
1033+
TVector<TString> filterWords = SplitString(Filter, " ");
1034+
TGroupView groupView;
1035+
for (TGroup* group : GroupView) {
1036+
for (const TString& word : filterWords) {
1037+
if (FieldsRequired.test(+EGroupFields::GroupId) && ::ToString(group->GroupId).Contains(word)) {
1038+
groupView.push_back(group);
1039+
break;
1040+
}
1041+
if (FieldsRequired.test(+EGroupFields::PoolName) && group->PoolName.Contains(word)) {
1042+
groupView.push_back(group);
1043+
break;
1044+
}
10531045
}
10541046
}
1047+
GroupView.swap(groupView);
1048+
Filter.clear();
1049+
GroupsByGroupId.clear();
10551050
}
1056-
GroupView.swap(groupView);
1057-
Filter.clear();
1058-
GroupsByGroupId.clear();
10591051
}
10601052
if (!FilterGroup.empty() && FieldsAvailable.test(+FilterGroupBy)) {
10611053
TGroupView groupView;
@@ -1220,6 +1212,7 @@ class TStorageGroups : public TViewerPipeClient {
12201212
bool CollectedHiveData = false;
12211213

12221214
void CollectHiveData() {
1215+
static TPathId badPathId(0, 0);
12231216
if (!CollectedHiveData) {
12241217
if (!GroupView.empty()) {
12251218
ui64 hiveId = AppData()->DomainsInfo->GetHive();
@@ -1232,6 +1225,9 @@ class TStorageGroups : public TViewerPipeClient {
12321225
}
12331226
for (const TGroup* group : GroupView) {
12341227
TPathId pathId(group->SchemeShardId, group->PathId);
1228+
if (pathId == badPathId) {
1229+
pathId = {AppData()->DomainsInfo->Domain->SchemeRoot, 1};
1230+
}
12351231
if (NavigateKeySetResult.count(pathId) == 0) {
12361232
ui64 cookie = NavigateKeySetResult.size();
12371233
NavigateKeySetResult.emplace(pathId, MakeRequestSchemeCacheNavigate(pathId, cookie));
@@ -1840,12 +1836,7 @@ class TStorageGroups : public TViewerPipeClient {
18401836
return;
18411837
}
18421838
if (BSGroupStateResponse.count(nodeId) == 0) {
1843-
TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId);
1844-
BSGroupStateResponse.emplace(nodeId, MakeRequest<TEvWhiteboard::TEvBSGroupStateResponse>(whiteboardServiceId,
1845-
new TEvWhiteboard::TEvBSGroupStateRequest(),
1846-
IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
1847-
nodeId));
1848-
SubscriptionNodeIds.push_back(nodeId);
1839+
BSGroupStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvBSGroupStateRequest()));
18491840
++BSGroupStateRequestsInFlight;
18501841
}
18511842
}
@@ -1854,22 +1845,13 @@ class TStorageGroups : public TViewerPipeClient {
18541845
if (nodeId == 0) {
18551846
return;
18561847
}
1857-
TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId);
18581848
if (VDiskStateResponse.count(nodeId) == 0) {
1859-
VDiskStateResponse.emplace(nodeId, MakeRequest<TEvWhiteboard::TEvVDiskStateResponse>(whiteboardServiceId,
1860-
new TEvWhiteboard::TEvVDiskStateRequest(),
1861-
IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
1862-
nodeId));
1849+
VDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvVDiskStateRequest()));
18631850
++VDiskStateRequestsInFlight;
1864-
SubscriptionNodeIds.push_back(nodeId);
18651851
}
18661852
if (PDiskStateResponse.count(nodeId) == 0) {
1867-
PDiskStateResponse.emplace(nodeId, MakeRequest<TEvWhiteboard::TEvPDiskStateResponse>(whiteboardServiceId,
1868-
new TEvWhiteboard::TEvPDiskStateRequest(),
1869-
IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
1870-
nodeId));
1853+
PDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvPDiskStateRequest()));
18711854
++PDiskStateRequestsInFlight;
1872-
SubscriptionNodeIds.push_back(nodeId);
18731855
}
18741856
}
18751857

‎ydb/core/viewer/viewer_autocomplete.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,6 @@ class TJsonAutocomplete : public TViewerPipeClient {
138138
TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult> MakeRequestSchemeCacheNavigate() {
139139
auto request = std::make_unique<NSchemeCache::TSchemeCacheNavigate>();
140140
for (const TString& path : Paths) {
141-
Cerr << "Looking into " << path << Endl;
142141
NSchemeCache::TSchemeCacheNavigate::TEntry entry;
143142
entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList;
144143
entry.SyncVersion = false;

0 commit comments

Comments
 (0)