diff --git a/ydb/core/base/pool_stats_collector.cpp b/ydb/core/base/pool_stats_collector.cpp index 7a06d5a3dd9f..40e57e8eb2ca 100644 --- a/ydb/core/base/pool_stats_collector.cpp +++ b/ydb/core/base/pool_stats_collector.cpp @@ -46,12 +46,24 @@ class TStatsCollectingActor : public NActors::TStatsCollectingActor { void OnWakeup(const TActorContext &ctx) override { MiniKQLPoolStats.Update(); - TVector> pools; + auto systemUpdate = std::make_unique(); + ui32 coresTotal = 0; + double coresUsed = 0; for (const auto& pool : PoolCounters) { - pools.emplace_back(pool.Name, pool.Usage, pool.Threads, pool.LimitThreads); + auto& pb = *systemUpdate->Record.AddPoolStats(); + pb.SetName(pool.Name); + pb.SetUsage(pool.Usage); + pb.SetThreads(static_cast(pool.Threads)); + pb.SetLimit(static_cast(pool.LimitThreads)); + if (pool.Name != "IO") { + coresTotal += static_cast(pool.DefaultThreads); + } + coresUsed += pool.Usage * pool.LimitThreads; } + systemUpdate->Record.SetCoresTotal(coresTotal); + systemUpdate->Record.SetCoresUsed(coresUsed); - ctx.Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(ctx.SelfID.NodeId()), new NNodeWhiteboard::TEvWhiteboard::TEvSystemStateUpdate(pools)); + ctx.Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(ctx.SelfID.NodeId()), systemUpdate.release()); } private: diff --git a/ydb/core/node_whiteboard/node_whiteboard.h b/ydb/core/node_whiteboard/node_whiteboard.h index a1cdd1a12d4f..ba0d5e999850 100644 --- a/ydb/core/node_whiteboard/node_whiteboard.h +++ b/ydb/core/node_whiteboard/node_whiteboard.h @@ -534,5 +534,8 @@ struct WhiteboardResponse { using Type = TEvWhiteboard::TEvNodeStateResponse; }; +template +::google::protobuf::RepeatedField GetDefaultWhiteboardFields(); + } // NNodeWhiteboard } // NKikimr diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto index fe5e2a6fafae..b41921fcef1d 100644 --- a/ydb/core/protos/node_whiteboard.proto +++ b/ydb/core/protos/node_whiteboard.proto @@ -319,7 +319,7 @@ message TSystemStateInfo { optional string DataCenter = 16; optional uint32 RackId = 17; optional string Rack = 18; - optional string Host = 19; + optional string Host = 19 [(DefaultField) = true]; optional string Version = 20 [(DefaultField) = true]; repeated TPoolStats PoolStats = 21 [(DefaultField) = true]; repeated TEndpoint Endpoints = 22 [(DefaultField) = true]; @@ -338,6 +338,8 @@ message TSystemStateInfo { optional TNodeSharedCache SharedCacheStats = 35; // TODO: use memory stats optional uint32 TotalSessions = 36 [(DefaultField) = true]; optional string NodeName = 37 [(DefaultField) = true]; + optional double CoresUsed = 39; + optional uint32 CoresTotal = 40; } message TEvSystemStateRequest { diff --git a/ydb/core/tablet/node_whiteboard.cpp b/ydb/core/tablet/node_whiteboard.cpp index 53468fec49d0..900f0e1134e0 100644 --- a/ydb/core/tablet/node_whiteboard.cpp +++ b/ydb/core/tablet/node_whiteboard.cpp @@ -477,7 +477,7 @@ class TNodeWhiteboardService : public TActorBootstrapped } } - static void SelectiveCopy(::google::protobuf::Message& protoTo, const ::google::protobuf::Message& protoFrom, const ::google::protobuf::RepeatedField& fields) { + static void SelectiveCopy(::google::protobuf::Message& protoTo, const ::google::protobuf::Message& protoFrom, const ::google::protobuf::RepeatedField& fields) { using namespace ::google::protobuf; const Descriptor& descriptor = *protoTo.GetDescriptor(); const Reflection& reflectionTo = *protoTo.GetReflection(); @@ -490,24 +490,6 @@ class TNodeWhiteboardService : public TActorBootstrapped } } - template - static ::google::protobuf::RepeatedField GetDefaultFields(const TMessage& message) { - using namespace ::google::protobuf; - const Descriptor& descriptor = *message.GetDescriptor(); - ::google::protobuf::RepeatedField defaultFields; - int fieldCount = descriptor.field_count(); - for (int index = 0; index < fieldCount; ++index) { - const FieldDescriptor* field = descriptor.field(index); - const auto& options(field->options()); - if (options.HasExtension(NKikimrWhiteboard::DefaultField)) { - if (options.GetExtension(NKikimrWhiteboard::DefaultField)) { - defaultFields.Add(field->number()); - } - } - } - return defaultFields; - } - template static void Copy(TMessage& to, const TMessage& from, const TRequest& request) { if (request.FieldsRequiredSize() > 0) { @@ -517,8 +499,7 @@ class TNodeWhiteboardService : public TActorBootstrapped SelectiveCopy(to, from, request.GetFieldsRequired()); } } else { - static auto defaultFields = GetDefaultFields(to); - SelectiveCopy(to, from, defaultFields); + SelectiveCopy(to, from, GetDefaultWhiteboardFields()); } } @@ -1121,6 +1102,30 @@ class TNodeWhiteboardService : public TActorBootstrapped } }; +template +::google::protobuf::RepeatedField InitDefaultWhiteboardFields() { + using namespace ::google::protobuf; + const Descriptor& descriptor = *TMessage::GetDescriptor(); + ::google::protobuf::RepeatedField defaultFields; + int fieldCount = descriptor.field_count(); + for (int index = 0; index < fieldCount; ++index) { + const FieldDescriptor* field = descriptor.field(index); + const auto& options(field->options()); + if (options.HasExtension(NKikimrWhiteboard::DefaultField)) { + if (options.GetExtension(NKikimrWhiteboard::DefaultField)) { + defaultFields.Add(field->number()); + } + } + } + return defaultFields; +} + +template +::google::protobuf::RepeatedField GetDefaultWhiteboardFields() { + static ::google::protobuf::RepeatedField defaultFields = InitDefaultWhiteboardFields(); + return defaultFields; +} + IActor* CreateNodeWhiteboardService() { return new TNodeWhiteboardService(); } diff --git a/ydb/core/viewer/json_handlers_viewer.cpp b/ydb/core/viewer/json_handlers_viewer.cpp index 56180de9dab3..b7f70577c398 100644 --- a/ydb/core/viewer/json_handlers_viewer.cpp +++ b/ydb/core/viewer/json_handlers_viewer.cpp @@ -203,7 +203,7 @@ void InitViewerStorageUsageJsonHandler(TJsonHandlers &handlers) { } void InitViewerClusterJsonHandler(TJsonHandlers& handlers) { - handlers.AddHandler("/viewer/cluster", new TJsonHandler(TJsonCluster::GetSwagger()), 4); + handlers.AddHandler("/viewer/cluster", new TJsonHandler(TJsonCluster::GetSwagger()), 5); } void InitViewerLabeledCountersJsonHandler(TJsonHandlers &handlers) { @@ -219,7 +219,7 @@ void InitViewerHiveStatsJsonHandler(TJsonHandlers& handlers) { } void InitViewerTenantInfoJsonHandler(TJsonHandlers &handlers) { - handlers.AddHandler("/viewer/tenantinfo", new TJsonHandler(TJsonTenantInfo::GetSwagger()), 2); + handlers.AddHandler("/viewer/tenantinfo", new TJsonHandler(TJsonTenantInfo::GetSwagger()), 3); } void InitViewerWhoAmIJsonHandler(TJsonHandlers& handlers) { @@ -227,7 +227,7 @@ void InitViewerWhoAmIJsonHandler(TJsonHandlers& handlers) { } void InitViewerQueryJsonHandler(TJsonHandlers& handlers) { - handlers.AddHandler("/viewer/query", new TJsonHandler(TJsonQuery::GetSwagger()), 3); + handlers.AddHandler("/viewer/query", new TJsonHandler(TJsonQuery::GetSwagger()), 4); } void InitViewerNetInfoJsonHandler(TJsonHandlers& handlers) { @@ -243,7 +243,7 @@ void InitViewerHealthCheckJsonHandler(TJsonHandlers& handlers) { } void InitViewerNodesJsonHandler(TJsonHandlers& handlers) { - handlers.AddHandler("/viewer/nodes", new TJsonHandler(TJsonNodes::GetSwagger()), 9); + handlers.AddHandler("/viewer/nodes", new TJsonHandler(TJsonNodes::GetSwagger()), 11); } void InitViewerACLJsonHandler(TJsonHandlers &jsonHandlers) { diff --git a/ydb/core/viewer/json_local_rpc.h b/ydb/core/viewer/json_local_rpc.h index df1ceafd792f..a2574bd6aa0f 100644 --- a/ydb/core/viewer/json_local_rpc.h +++ b/ydb/core/viewer/json_local_rpc.h @@ -128,10 +128,9 @@ class TJsonLocalRpc : public TViewerPipeClient { ReplyAndPassAway(GetHTTPBADREQUEST("text/plain", e.what())); return false; } - } else { - const auto& params(Event->Get()->Request.GetParams()); - Params2Proto(params, request); } + const auto& params(Event->Get()->Request.GetParams()); + Params2Proto(params, request); if (!ValidateRequest(request)) { return false; } diff --git a/ydb/core/viewer/json_pipe_req.cpp b/ydb/core/viewer/json_pipe_req.cpp index b2fc57d9b28a..52777ea8492c 100644 --- a/ydb/core/viewer/json_pipe_req.cpp +++ b/ydb/core/viewer/json_pipe_req.cpp @@ -1,4 +1,5 @@ #include "json_pipe_req.h" +#include #include namespace NKikimr::NViewer { @@ -25,7 +26,28 @@ TViewerPipeClient::TViewerPipeClient(IViewer* viewer, NMon::TEvHttpInfo::TPtr& e : Viewer(viewer) , Event(ev) { - InitConfig(Event->Get()->Request.GetParams()); + TCgiParameters params = Event->Get()->Request.GetParams(); + if (Event->Get()->Request.GetHeader("Content-Type") == "application/json") { + NJson::TJsonValue jsonData; + if (NJson::ReadJsonTree(Event->Get()->Request.GetPostContent(), &jsonData)) { + if (jsonData.IsMap()) { + for (const auto& [key, value] : jsonData.GetMap()) { + switch (value.GetType()) { + case NJson::EJsonValueType::JSON_STRING: + case NJson::EJsonValueType::JSON_INTEGER: + case NJson::EJsonValueType::JSON_UINTEGER: + case NJson::EJsonValueType::JSON_DOUBLE: + case NJson::EJsonValueType::JSON_BOOLEAN: + params.InsertUnescaped(key, value.GetStringRobust()); + break; + default: + break; + } + } + } + } + } + InitConfig(params); NWilson::TTraceId traceId; TStringBuf traceparent = Event->Get()->Request.GetHeader("traceparent"); if (traceparent) { @@ -645,10 +667,20 @@ TRequestState TViewerPipeClient::GetRequest() const { } void TViewerPipeClient::ReplyAndPassAway(TString data, const TString& error) { + TString message = error; Send(Event->Sender, new NMon::TEvHttpInfoRes(data, 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + if (message.empty()) { + TStringBuf dataParser(data); + if (dataParser.NextTok(' ') == "HTTP/1.1") { + TStringBuf code = dataParser.NextTok(' '); + if (code.size() == 3 && code[0] != '2') { + message = dataParser.NextTok('\n'); + } + } + } if (Span) { - if (error) { - Span.EndError(error); + if (message) { + Span.EndError(message); } else { Span.EndOk(); } diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index a88f66663007..6ad06ff4e428 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -322,6 +322,7 @@ message TClusterInfo { uint32 NodesTotal = 10; uint32 NodesAlive = 11; uint32 NumberOfCpus = 20; + uint32 CoresTotal = 24; double CoresUsed = 21; double LoadAverage = 22; repeated NKikimrWhiteboard.TSystemStateInfo.TPoolStats PoolStats = 23; @@ -358,7 +359,7 @@ message TStorageUsage { SSD = 2; } EType Type = 1; - uint64 Size = 2; + optional uint64 Size = 2; uint64 Limit = 3; uint64 SoftQuota = 4; uint64 HardQuota = 5; @@ -394,6 +395,7 @@ message TTenant { Ydb.Cms.DatabaseQuotas DatabaseQuotas = 42; repeated TStorageUsage TablesStorage = 44; repeated TStorageUsage DatabaseStorage = 45; + uint32 CoresTotal = 50; } message TTenants { diff --git a/ydb/core/viewer/storage_groups.h b/ydb/core/viewer/storage_groups.h index 4469239ccee6..6c2d28e07354 100644 --- a/ydb/core/viewer/storage_groups.h +++ b/ydb/core/viewer/storage_groups.h @@ -151,7 +151,6 @@ class TStorageGroups : public TViewerPipeClient { std::unordered_set FilterGroupIds; std::unordered_set FilterNodeIds; std::unordered_set FilterPDiskIds; - std::vector SubscriptionNodeIds; enum class EWith { Everything, @@ -894,22 +893,6 @@ class TStorageGroups : public TViewerPipeClient { Schedule(TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup(TimeoutFinal)); // timeout for the rest } - void PassAway() override { - std::vector passedNodes; - for (const TNodeId nodeId : SubscriptionNodeIds) { - if (passedNodes.size() <= nodeId) { - passedNodes.resize(nodeId + 1); - } else { - if (passedNodes[nodeId]) { - continue; - } - } - Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe()); - passedNodes[nodeId] = true; - } - TBase::PassAway(); - } - void ApplyFilter() { // database pre-filter, affects TotalGroups count if (!DatabaseStoragePools.empty()) { @@ -1042,20 +1025,29 @@ class TStorageGroups : public TViewerPipeClient { With = EWith::Everything; GroupsByGroupId.clear(); } - if (!Filter.empty() && FieldsAvailable.test(+EGroupFields::PoolName) && FieldsAvailable.test(+EGroupFields::GroupId)) { - TVector filterWords = SplitString(Filter, " "); - TGroupView groupView; - for (TGroup* group : GroupView) { - for (const TString& word : filterWords) { - if (group->PoolName.Contains(word) || ::ToString(group->GroupId).Contains(word)) { - groupView.push_back(group); - break; + if (!Filter.empty()) { + bool allFieldsPresent = + (!FieldsRequired.test(+EGroupFields::GroupId) || FieldsAvailable.test(+EGroupFields::GroupId)) && + (!FieldsRequired.test(+EGroupFields::PoolName) || FieldsAvailable.test(+EGroupFields::PoolName)); + if (allFieldsPresent) { + TVector filterWords = SplitString(Filter, " "); + TGroupView groupView; + for (TGroup* group : GroupView) { + for (const TString& word : filterWords) { + if (FieldsRequired.test(+EGroupFields::GroupId) && ::ToString(group->GroupId).Contains(word)) { + groupView.push_back(group); + break; + } + if (FieldsRequired.test(+EGroupFields::PoolName) && group->PoolName.Contains(word)) { + groupView.push_back(group); + break; + } } } + GroupView.swap(groupView); + Filter.clear(); + GroupsByGroupId.clear(); } - GroupView.swap(groupView); - Filter.clear(); - GroupsByGroupId.clear(); } if (!FilterGroup.empty() && FieldsAvailable.test(+FilterGroupBy)) { TGroupView groupView; @@ -1220,6 +1212,7 @@ class TStorageGroups : public TViewerPipeClient { bool CollectedHiveData = false; void CollectHiveData() { + static TPathId badPathId(0, 0); if (!CollectedHiveData) { if (!GroupView.empty()) { ui64 hiveId = AppData()->DomainsInfo->GetHive(); @@ -1232,6 +1225,9 @@ class TStorageGroups : public TViewerPipeClient { } for (const TGroup* group : GroupView) { TPathId pathId(group->SchemeShardId, group->PathId); + if (pathId == badPathId) { + pathId = {AppData()->DomainsInfo->Domain->SchemeRoot, 1}; + } if (NavigateKeySetResult.count(pathId) == 0) { ui64 cookie = NavigateKeySetResult.size(); NavigateKeySetResult.emplace(pathId, MakeRequestSchemeCacheNavigate(pathId, cookie)); @@ -1840,12 +1836,7 @@ class TStorageGroups : public TViewerPipeClient { return; } if (BSGroupStateResponse.count(nodeId) == 0) { - TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId); - BSGroupStateResponse.emplace(nodeId, MakeRequest(whiteboardServiceId, - new TEvWhiteboard::TEvBSGroupStateRequest(), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, - nodeId)); - SubscriptionNodeIds.push_back(nodeId); + BSGroupStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvBSGroupStateRequest())); ++BSGroupStateRequestsInFlight; } } @@ -1854,22 +1845,13 @@ class TStorageGroups : public TViewerPipeClient { if (nodeId == 0) { return; } - TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId); if (VDiskStateResponse.count(nodeId) == 0) { - VDiskStateResponse.emplace(nodeId, MakeRequest(whiteboardServiceId, - new TEvWhiteboard::TEvVDiskStateRequest(), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, - nodeId)); + VDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvVDiskStateRequest())); ++VDiskStateRequestsInFlight; - SubscriptionNodeIds.push_back(nodeId); } if (PDiskStateResponse.count(nodeId) == 0) { - PDiskStateResponse.emplace(nodeId, MakeRequest(whiteboardServiceId, - new TEvWhiteboard::TEvPDiskStateRequest(), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, - nodeId)); + PDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvPDiskStateRequest())); ++PDiskStateRequestsInFlight; - SubscriptionNodeIds.push_back(nodeId); } } diff --git a/ydb/core/viewer/viewer_autocomplete.h b/ydb/core/viewer/viewer_autocomplete.h index 069f322532c3..36b29974d9f5 100644 --- a/ydb/core/viewer/viewer_autocomplete.h +++ b/ydb/core/viewer/viewer_autocomplete.h @@ -138,7 +138,6 @@ class TJsonAutocomplete : public TViewerPipeClient { TRequestResponse MakeRequestSchemeCacheNavigate() { auto request = std::make_unique(); for (const TString& path : Paths) { - Cerr << "Looking into " << path << Endl; NSchemeCache::TSchemeCacheNavigate::TEntry entry; entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList; entry.SyncVersion = false; diff --git a/ydb/core/viewer/viewer_capabilities.h b/ydb/core/viewer/viewer_capabilities.h index 3804ecbede94..c310be4945ce 100644 --- a/ydb/core/viewer/viewer_capabilities.h +++ b/ydb/core/viewer/viewer_capabilities.h @@ -15,6 +15,9 @@ class TViewerCapabilities : public TViewerPipeClient { {} void Bootstrap() override { + if (TBase::NeedToRedirect()) { + return; + } ReplyAndPassAway(); } diff --git a/ydb/core/viewer/viewer_cluster.h b/ydb/core/viewer/viewer_cluster.h index 788363ba5ea0..b74936c3c304 100644 --- a/ydb/core/viewer/viewer_cluster.h +++ b/ydb/core/viewer/viewer_cluster.h @@ -342,8 +342,9 @@ class TJsonCluster : public TViewerPipeClient { } void InitSystemWhiteboardRequest(NKikimrWhiteboard::TEvSystemStateRequest* request) { - //request->AddFieldsRequired(-1); - Y_UNUSED(request); + request->MutableFieldsRequired()->CopyFrom(GetDefaultWhiteboardFields()); + request->AddFieldsRequired(NKikimrWhiteboard::TSystemStateInfo::kCoresUsedFieldNumber); + request->AddFieldsRequired(NKikimrWhiteboard::TSystemStateInfo::kCoresTotalFieldNumber); } void InitTabletWhiteboardRequest(NKikimrWhiteboard::TEvTabletStateRequest* request) { @@ -462,14 +463,16 @@ class TJsonCluster : public TViewerPipeClient { } } + std::unordered_set hostPassed; + for (TNode& node : NodeData) { const NKikimrWhiteboard::TSystemStateInfo& systemState = node.SystemState; (*ClusterInfo.MutableMapDataCenters())[node.DataCenter]++; - if (systemState.HasNumberOfCpus()) { + if (hostPassed.insert(systemState.GetHost()).second) { ClusterInfo.SetNumberOfCpus(ClusterInfo.GetNumberOfCpus() + systemState.GetNumberOfCpus()); - } - if (systemState.LoadAverageSize() > 0) { - ClusterInfo.SetLoadAverage(ClusterInfo.GetLoadAverage() + systemState.GetLoadAverage(0)); + if (systemState.LoadAverageSize() > 0) { + ClusterInfo.SetLoadAverage(ClusterInfo.GetLoadAverage() + systemState.GetLoadAverage(0)); + } } if (systemState.HasVersion()) { (*ClusterInfo.MutableMapVersions())[systemState.GetVersion()]++; @@ -477,8 +480,8 @@ class TJsonCluster : public TViewerPipeClient { if (systemState.HasClusterName() && !ClusterInfo.GetName()) { ClusterInfo.SetName(systemState.GetClusterName()); } - ClusterInfo.SetMemoryTotal(ClusterInfo.GetMemoryTotal() + systemState.GetMemoryLimit()); ClusterInfo.SetMemoryUsed(ClusterInfo.GetMemoryUsed() + systemState.GetMemoryUsed()); + ClusterInfo.SetMemoryTotal(ClusterInfo.GetMemoryTotal() + systemState.GetMemoryLimit()); if (!node.Disconnected && node.SystemState.HasSystemState()) { ClusterInfo.SetNodesAlive(ClusterInfo.GetNodesAlive() + 1); } @@ -500,14 +503,24 @@ class TJsonCluster : public TViewerPipeClient { targetPoolStat->SetName(poolName); } double poolUsage = targetPoolStat->GetUsage() * targetPoolStat->GetThreads(); - poolUsage += poolStat.GetUsage() * poolStat.GetThreads(); + ui32 usageThreads = poolStat.GetLimit() ? poolStat.GetLimit() : poolStat.GetThreads(); + poolUsage += poolStat.GetUsage() * usageThreads; ui32 poolThreads = targetPoolStat->GetThreads() + poolStat.GetThreads(); if (poolThreads != 0) { double threadUsage = poolUsage / poolThreads; targetPoolStat->SetUsage(threadUsage); targetPoolStat->SetThreads(poolThreads); } - ClusterInfo.SetCoresUsed(ClusterInfo.GetCoresUsed() + poolStat.GetUsage() * poolStat.GetThreads()); + if (systemState.GetCoresTotal() == 0) { + ClusterInfo.SetCoresUsed(ClusterInfo.GetCoresUsed() + poolStat.GetUsage() * usageThreads); + if (poolStat.GetName() != "IO") { + ClusterInfo.SetCoresTotal(ClusterInfo.GetCoresTotal() + poolStat.GetThreads()); + } + } + } + if (systemState.GetCoresTotal() != 0) { + ClusterInfo.SetCoresUsed(ClusterInfo.GetCoresUsed() + systemState.GetCoresUsed()); + ClusterInfo.SetCoresTotal(ClusterInfo.GetCoresTotal() + systemState.GetCoresTotal()); } } diff --git a/ydb/core/viewer/viewer_nodes.h b/ydb/core/viewer/viewer_nodes.h index 6ab28f8798a3..ff8a82a4e4ce 100644 --- a/ydb/core/viewer/viewer_nodes.h +++ b/ydb/core/viewer/viewer_nodes.h @@ -260,11 +260,18 @@ class TJsonNodes : public TViewerPipeClient { } void CalcCpuUsage() { - float usage = 0; - int threads = 0; - for (const auto& pool : SystemState.GetPoolStats()) { - usage += pool.GetUsage() * pool.GetThreads(); - threads += pool.GetThreads(); + float usage = SystemState.GetCoresUsed(); + int threads = SystemState.GetCoresTotal(); + if (threads == 0) { + for (const auto& pool : SystemState.GetPoolStats()) { + ui32 usageThreads = pool.GetLimit() ? pool.GetLimit() : pool.GetThreads(); + usage += pool.GetUsage() * usageThreads; + if (pool.GetName() != "IO") { + threads += pool.GetThreads(); + } + } + SystemState.SetCoresUsed(usage); + SystemState.SetCoresTotal(threads); } CpuUsage = usage / threads; } @@ -620,7 +627,7 @@ class TJsonNodes : public TViewerPipeClient { ENodeFields result = ENodeFields::COUNT; if (field == "NodeId" || field == "Id") { result = ENodeFields::NodeId; - } else if (field == "Host") { + } else if (field == "Host" || field == "HostName") { result = ENodeFields::HostName; } else if (field == "NodeName") { result = ENodeFields::NodeName; @@ -1023,20 +1030,34 @@ class TJsonNodes : public TViewerPipeClient { UptimeSeconds = 0; InvalidateNodes(); } - if (!Filter.empty() && FieldsAvailable.test(+ENodeFields::NodeInfo)) { - TVector filterWords = SplitString(Filter, " "); - TNodeView nodeView; - for (TNode* node : NodeView) { - for (const TString& word : filterWords) { - if (node->GetHostName().Contains(word) || ::ToString(node->GetNodeId()).Contains(word)) { - nodeView.push_back(node); - break; + if (!Filter.empty()) { + bool allFieldsPresent = + (!FieldsRequired.test(+ENodeFields::NodeId) || FieldsAvailable.test(+ENodeFields::NodeId)) && + (!FieldsRequired.test(+ENodeFields::HostName) || FieldsAvailable.test(+ENodeFields::HostName)) && + (!FieldsRequired.test(+ENodeFields::NodeName) || FieldsAvailable.test(+ENodeFields::NodeName)); + if (allFieldsPresent) { + TVector filterWords = SplitString(Filter, " "); + TNodeView nodeView; + for (TNode* node : NodeView) { + for (const TString& word : filterWords) { + if (FieldsRequired.test(+ENodeFields::NodeId) && ::ToString(node->GetNodeId()).Contains(word)) { + nodeView.push_back(node); + break; + } + if (FieldsRequired.test(+ENodeFields::HostName) && node->GetHostName().Contains(word)) { + nodeView.push_back(node); + break; + } + if (FieldsRequired.test(+ENodeFields::NodeName) && node->GetNodeName().Contains(word)) { + nodeView.push_back(node); + break; + } } } + NodeView.swap(nodeView); + Filter.clear(); + InvalidateNodes(); } - NodeView.swap(nodeView); - Filter.clear(); - InvalidateNodes(); } if (!FilterGroup.empty() && FieldsAvailable.test(+FilterGroupBy)) { TNodeView nodeView; @@ -1621,7 +1642,7 @@ class TJsonNodes : public TViewerPipeClient { } GroupsResponse.reset(); } - if (FilterStorageStage == EFilterStorageStage::VSlots && VSlotsResponse && VSlotsResponse->IsDone()) { + if ((FilterStorageStage == EFilterStorageStage::VSlots || FilterStorageStage == EFilterStorageStage::None) && VSlotsResponse && VSlotsResponse->IsDone()) { if (VSlotsResponse->IsOk()) { std::unordered_set prevFilterNodeIds = std::move(FilterNodeIds); std::unordered_map, std::size_t> slotsPerDisk; @@ -1695,6 +1716,17 @@ class TJsonNodes : public TViewerPipeClient { } } + template<> + void InitWhiteboardRequest(NKikimrWhiteboard::TEvSystemStateRequest* request) { + if (AllWhiteboardFields) { + request->AddFieldsRequired(-1); + } else { + request->MutableFieldsRequired()->CopyFrom(GetDefaultWhiteboardFields()); + request->AddFieldsRequired(NKikimrWhiteboard::TSystemStateInfo::kCoresUsedFieldNumber); + request->AddFieldsRequired(NKikimrWhiteboard::TSystemStateInfo::kCoresTotalFieldNumber); + } + } + void SendWhiteboardSystemAndTabletsBatch(TNodeBatch& batch) { TNodeId nodeId = OffloadMerge ? batch.ChooseNodeId() : 0; if (batch.HasStaticNodes && (FieldsNeeded(FieldsVDisks) || FieldsNeeded(FieldsPDisks))) { diff --git a/ydb/core/viewer/viewer_query.h b/ydb/core/viewer/viewer_query.h index 75292362865c..d256a1a49a31 100644 --- a/ydb/core/viewer/viewer_query.h +++ b/ydb/core/viewer/viewer_query.h @@ -37,6 +37,7 @@ class TJsonQuery : public TViewerPipeClient { Modern, Multi, Ydb, + Ydb2, }; ESchemaType Schema = ESchemaType::Classic; TRequestResponse CreateSessionResponse; @@ -53,6 +54,8 @@ class TJsonQuery : public TViewerPipeClient { return ESchemaType::Multi; } else if (schemaStr == "ydb") { return ESchemaType::Ydb; + } else if (schemaStr == "ydb2") { + return ESchemaType::Ydb2; } else { return ESchemaType::Classic; } @@ -393,8 +396,8 @@ class TJsonQuery : public TViewerPipeClient { return valueParser.GetDyNumber(); case NYdb::EPrimitiveType::Uuid: return valueParser.GetUuid().ToString(); - default: - Y_ENSURE(false, TStringBuilder() << "Unsupported type: " << primitive); } + } + return NJson::JSON_UNDEFINED; } NJson::TJsonValue ColumnValueToJsonValue(NYdb::TValueParser& valueParser) { @@ -426,6 +429,63 @@ class TJsonQuery : public TViewerPipeClient { case NYdb::TTypeParser::ETypeKind::Decimal: return valueParser.GetDecimal().ToString(); + case NYdb::TTypeParser::ETypeKind::List: + { + NJson::TJsonValue jsonList; + jsonList.SetType(NJson::JSON_ARRAY); + valueParser.OpenList(); + while (valueParser.TryNextListItem()) { + jsonList.AppendValue(ColumnValueToJsonValue(valueParser)); + } + return jsonList; + } + + case NYdb::TTypeParser::ETypeKind::Tuple: + { + NJson::TJsonValue jsonTuple; + jsonTuple.SetType(NJson::JSON_ARRAY); + valueParser.OpenTuple(); + while (valueParser.TryNextElement()) { + jsonTuple.AppendValue(ColumnValueToJsonValue(valueParser)); + } + return jsonTuple; + } + + case NYdb::TTypeParser::ETypeKind::Struct: + { + NJson::TJsonValue jsonStruct; + jsonStruct.SetType(NJson::JSON_MAP); + valueParser.OpenStruct(); + while (valueParser.TryNextMember()) { + jsonStruct[valueParser.GetMemberName()] = ColumnValueToJsonValue(valueParser); + } + return jsonStruct; + } + + case NYdb::TTypeParser::ETypeKind::Dict: + { + NJson::TJsonValue jsonDict; + jsonDict.SetType(NJson::JSON_MAP); + valueParser.OpenDict(); + while (valueParser.TryNextDictItem()) { + valueParser.DictKey(); + TString key = valueParser.GetString(); + valueParser.DictPayload(); + jsonDict[key] = ColumnValueToJsonValue(valueParser); + } + return jsonDict; + } + + case NYdb::TTypeParser::ETypeKind::Variant: + valueParser.OpenVariant(); + return ColumnValueToJsonValue(valueParser); + + case NYdb::TTypeParser::ETypeKind::EmptyList: + return NJson::JSON_ARRAY; + + case NYdb::TTypeParser::ETypeKind::EmptyDict: + return NJson::JSON_MAP; + default: return NJson::JSON_UNDEFINED; } @@ -664,6 +724,37 @@ class TJsonQuery : public TViewerPipeClient { } } } + + if (Schema == ESchemaType::Ydb2) { + NJson::TJsonValue& jsonResults = jsonResponse["result"]; + jsonResults.SetType(NJson::JSON_ARRAY); + for (const auto& resultSets : ResultSets) { + NJson::TJsonValue& jsonResult = jsonResults.AppendValue({}); + bool hasColumns = false; + for (NYdb::TResultSet resultSet : resultSets) { + if (!hasColumns) { + NJson::TJsonValue& jsonColumns = jsonResult["columns"]; + jsonColumns.SetType(NJson::JSON_ARRAY); + const auto& columnsMeta = resultSet.GetColumnsMeta(); + for (size_t columnNum = 0; columnNum < columnsMeta.size(); ++columnNum) { + NJson::TJsonValue& jsonColumn = jsonColumns.AppendValue({}); + const NYdb::TColumn& columnMeta = columnsMeta[columnNum]; + jsonColumn["name"] = columnMeta.Name; + jsonColumn["type"] = columnMeta.Type.ToString(); + } + hasColumns = true; + } + NJson::TJsonValue& jsonRows = jsonResult["rows"]; + const auto& columnsMeta = resultSet.GetColumnsMeta(); + NYdb::TResultSetParser rsParser(resultSet); + while (rsParser.TryNextRow()) { + NJson::TJsonValue& jsonRow = jsonRows.AppendValue({}); + TString row = NYdb::FormatResultRowJson(rsParser, columnsMeta, IsBase64Encode ? NYdb::EBinaryStringEncoding::Base64 : NYdb::EBinaryStringEncoding::Unicode); + NJson::ReadJsonTree(row, &jsonRow); + } + } + } + } } if (response.HasQueryAst()) { jsonResponse["ast"] = response.GetQueryAst(); diff --git a/ydb/core/viewer/viewer_tenantinfo.h b/ydb/core/viewer/viewer_tenantinfo.h index 887a1521204d..480803559385 100644 --- a/ydb/core/viewer/viewer_tenantinfo.h +++ b/ydb/core/viewer/viewer_tenantinfo.h @@ -262,7 +262,11 @@ class TJsonTenantInfo : public TViewerPipeClient { void SendWhiteboardSystemStateRequest(const TNodeId nodeId) { Subscribers.insert(nodeId); if (SystemStateResponse.count(nodeId) == 0) { - SystemStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvSystemStateRequest())); + auto request = std::make_unique(); + request->Record.MutableFieldsRequired()->CopyFrom(GetDefaultWhiteboardFields()); + request->Record.AddFieldsRequired(NKikimrWhiteboard::TSystemStateInfo::kCoresUsedFieldNumber); + request->Record.AddFieldsRequired(NKikimrWhiteboard::TSystemStateInfo::kCoresTotalFieldNumber); + SystemStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, request.release())); } } @@ -731,7 +735,8 @@ class TJsonTenantInfo : public TViewerPipeClient { if (tablesStorageByType.empty() && entry.DomainDescription->Description.HasDiskSpaceUsage()) { tablesStorageByType[GuessStorageType(entry.DomainDescription->Description)] = - entry.DomainDescription->Description.GetDiskSpaceUsage().GetTables().GetTotalSize(); + entry.DomainDescription->Description.GetDiskSpaceUsage().GetTables().GetTotalSize() + + entry.DomainDescription->Description.GetDiskSpaceUsage().GetTopics().GetDataSize(); } if (storageQuotasByType.empty()) { @@ -784,14 +789,24 @@ class TJsonTenantInfo : public TViewerPipeClient { targetPoolStat->SetName(poolName); } double poolUsage = targetPoolStat->GetUsage() * targetPoolStat->GetThreads(); - poolUsage += poolStat.GetUsage() * poolStat.GetThreads(); + ui32 usageThreads = poolStat.GetLimit() ? poolStat.GetLimit() : poolStat.GetThreads(); + poolUsage += poolStat.GetUsage() * usageThreads; ui32 poolThreads = targetPoolStat->GetThreads() + poolStat.GetThreads(); if (poolThreads != 0) { double threadUsage = poolUsage / poolThreads; targetPoolStat->SetUsage(threadUsage); targetPoolStat->SetThreads(poolThreads); } - tenant.SetCoresUsed(tenant.GetCoresUsed() + poolStat.GetUsage() * poolStat.GetThreads()); + if (nodeInfo.GetCoresTotal() == 0) { + tenant.SetCoresUsed(tenant.GetCoresUsed() + poolStat.GetUsage() * usageThreads); + if (poolStat.GetName() != "IO") { + tenant.SetCoresTotal(tenant.GetCoresTotal() + poolStat.GetThreads()); + } + } + } + if (nodeInfo.GetCoresTotal() > 0) { + tenant.SetCoresUsed(tenant.GetCoresUsed() + nodeInfo.GetCoresUsed()); + tenant.SetCoresTotal(tenant.GetCoresTotal() + nodeInfo.GetCoresTotal()); } if (nodeInfo.HasMemoryUsed()) { tenant.SetMemoryUsed(tenant.GetMemoryUsed() + nodeInfo.GetMemoryUsed()); diff --git a/ydb/library/actors/helpers/pool_stats_collector.h b/ydb/library/actors/helpers/pool_stats_collector.h index fb4c3d9286b8..a0790a55c8bf 100644 --- a/ydb/library/actors/helpers/pool_stats_collector.h +++ b/ydb/library/actors/helpers/pool_stats_collector.h @@ -215,6 +215,7 @@ class TStatsCollectingActor : public TActorBootstrapped { TString Name; double Threads; double LimitThreads; + double DefaultThreads; void Init(NMonitoring::TDynamicCounters* group, const TString& poolName, ui32 threads) { LastElapsedSeconds = 0; @@ -223,6 +224,7 @@ class TStatsCollectingActor : public TActorBootstrapped { Name = poolName; Threads = threads; LimitThreads = threads; + DefaultThreads = threads; PoolGroup = group->GetSubgroup("execpool", poolName); @@ -292,6 +294,7 @@ class TStatsCollectingActor : public TActorBootstrapped { void Set(const TExecutorPoolStats& poolStats, const TExecutorThreadStats& stats) { #ifdef ACTORSLIB_COLLECT_EXEC_STATS + double elapsedSeconds = ::NHPTimer::GetSeconds(stats.ElapsedTicks); *SentEvents = stats.SentEvents; *ReceivedEvents = stats.ReceivedEvents; *PreemptedEvents = stats.PreemptedEvents; @@ -299,7 +302,7 @@ class TStatsCollectingActor : public TActorBootstrapped { *DestroyedActors = stats.PoolDestroyedActors; *EmptyMailboxActivation = stats.EmptyMailboxActivation; *CpuMicrosec = stats.CpuUs; - *ElapsedMicrosec = ::NHPTimer::GetSeconds(stats.ElapsedTicks)*1000000; + *ElapsedMicrosec = elapsedSeconds*1000000; *ParkedMicrosec = ::NHPTimer::GetSeconds(stats.ParkedTicks)*1000000; *ActorRegistrations = stats.PoolActorRegistrations; *ActorsAlive = stats.PoolActorRegistrations - stats.PoolDestroyedActors; @@ -364,19 +367,17 @@ class TStatsCollectingActor : public TActorBootstrapped { double seconds = UsageTimer.PassedReset(); // TODO[serxa]: It doesn't account for contention. Use 1 - parkedTicksDelta / seconds / numThreads KIKIMR-11916 - const double currentThreadCount = poolStats.PotentialMaxThreadCount; - const double elapsed = NHPTimer::GetSeconds(stats.ElapsedTicks); - const double currentUsage = currentThreadCount > 0 ? ((elapsed - LastElapsedSeconds) / seconds / currentThreadCount) : 0; - LastElapsedSeconds = elapsed; + Threads = poolStats.CurrentThreadCount; + LimitThreads = poolStats.PotentialMaxThreadCount; + const double currentUsage = LimitThreads > 0 ? ((elapsedSeconds - LastElapsedSeconds) / seconds / LimitThreads) : 0; // update usage factor according to smoothness const double smoothness = 0.5; Usage = currentUsage * smoothness + Usage * (1.0 - smoothness); + LastElapsedSeconds = elapsedSeconds; #else Y_UNUSED(stats); #endif - Threads = poolStats.CurrentThreadCount; - LimitThreads = poolStats.PotentialMaxThreadCount; } }; diff --git a/ydb/public/api/protos/ydb_scheme.proto b/ydb/public/api/protos/ydb_scheme.proto index 4ddcafc40a04..e7ddf9ce2818 100644 --- a/ydb/public/api/protos/ydb_scheme.proto +++ b/ydb/public/api/protos/ydb_scheme.proto @@ -5,6 +5,7 @@ package Ydb.Scheme; option java_package = "com.yandex.ydb.scheme"; option java_outer_classname = "SchemeOperationProtos"; +import "ydb/public/api/protos/annotations/validation.proto"; import "ydb/public/api/protos/ydb_common.proto"; import "ydb/public/api/protos/ydb_operation.proto"; @@ -12,7 +13,7 @@ import "ydb/public/api/protos/ydb_operation.proto"; // All intermediate directories must be created message MakeDirectoryRequest { Ydb.Operations.OperationParams operation_params = 1; - string path = 2; + string path = 2 [(required) = true]; } message MakeDirectoryResponse { @@ -22,7 +23,7 @@ message MakeDirectoryResponse { // Remove directory message RemoveDirectoryRequest { Ydb.Operations.OperationParams operation_params = 1; - string path = 2; + string path = 2 [(required) = true]; } message RemoveDirectoryResponse { @@ -32,7 +33,7 @@ message RemoveDirectoryResponse { // List directory message ListDirectoryRequest { Ydb.Operations.OperationParams operation_params = 1; - string path = 2; + string path = 2 [(required) = true]; } message ListDirectoryResponse { @@ -93,7 +94,7 @@ message ListDirectoryResult { // Returns information about object with given path message DescribePathRequest { Ydb.Operations.OperationParams operation_params = 1; - string path = 2; + string path = 2 [(required) = true]; } message DescribePathResponse { @@ -121,7 +122,7 @@ message PermissionsAction { // Modify permissions of given object message ModifyPermissionsRequest { Ydb.Operations.OperationParams operation_params = 1; - string path = 2; + string path = 2 [(required) = true]; repeated PermissionsAction actions = 3; // Clear all permissions on the object for all subjects bool clear_permissions = 4;