Viewer updates stable 24-3-hotfix-11

ydb/core/base/pool_stats_collector.cpp

@@ -46,12 +46,24 @@ class TStatsCollectingActor : public NActors::TStatsCollectingActor {
     void OnWakeup(const TActorContext &ctx) override {
         MiniKQLPoolStats.Update();
 
-        TVector<std::tuple<TString, double, ui32, ui32>> pools;
+        auto systemUpdate = std::make_unique<NNodeWhiteboard::TEvWhiteboard::TEvSystemStateUpdate>();
+        ui32 coresTotal = 0;
+        double coresUsed = 0;
         for (const auto& pool : PoolCounters) {
-            pools.emplace_back(pool.Name, pool.Usage, pool.Threads, pool.LimitThreads);
+            auto& pb = *systemUpdate->Record.AddPoolStats();
+            pb.SetName(pool.Name);
+            pb.SetUsage(pool.Usage);
+            pb.SetThreads(static_cast<ui32>(pool.Threads));
+            pb.SetLimit(static_cast<ui32>(pool.LimitThreads));
+            if (pool.Name != "IO") {
+                coresTotal += static_cast<ui32>(pool.DefaultThreads);
+            }
+            coresUsed += pool.Usage * pool.LimitThreads;
         }
+        systemUpdate->Record.SetCoresTotal(coresTotal);
+        systemUpdate->Record.SetCoresUsed(coresUsed);
 
-        ctx.Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(ctx.SelfID.NodeId()), new NNodeWhiteboard::TEvWhiteboard::TEvSystemStateUpdate(pools));
+        ctx.Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(ctx.SelfID.NodeId()), systemUpdate.release());
     }
 
 private:

ydb/core/node_whiteboard/node_whiteboard.h

@@ -534,5 +534,8 @@ struct WhiteboardResponse<TEvWhiteboard::TEvNodeStateRequest> {
     using Type = TEvWhiteboard::TEvNodeStateResponse;
 };
 
+template<typename TResponseType>
+::google::protobuf::RepeatedField<int> GetDefaultWhiteboardFields();
+
 } // NNodeWhiteboard
 } // NKikimr

ydb/core/protos/node_whiteboard.proto

@@ -319,7 +319,7 @@ message TSystemStateInfo {
     optional string DataCenter = 16;
     optional uint32 RackId = 17;
     optional string Rack = 18;
-    optional string Host = 19;
+    optional string Host = 19 [(DefaultField) = true];
     optional string Version = 20 [(DefaultField) = true];
     repeated TPoolStats PoolStats = 21 [(DefaultField) = true];
     repeated TEndpoint Endpoints = 22 [(DefaultField) = true];
@@ -338,6 +338,8 @@ message TSystemStateInfo {
     optional TNodeSharedCache SharedCacheStats = 35; // TODO: use memory stats
     optional uint32 TotalSessions = 36 [(DefaultField) = true];
     optional string NodeName = 37 [(DefaultField) = true];
+    optional double CoresUsed = 39;
+    optional uint32 CoresTotal = 40;
 }
 
 message TEvSystemStateRequest {

ydb/core/tablet/node_whiteboard.cpp

@@ -477,7 +477,7 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
         }
     }
 
-    static void SelectiveCopy(::google::protobuf::Message& protoTo, const ::google::protobuf::Message& protoFrom, const ::google::protobuf::RepeatedField<arc_i32>& fields) {
+    static void SelectiveCopy(::google::protobuf::Message& protoTo, const ::google::protobuf::Message& protoFrom, const ::google::protobuf::RepeatedField<int>& fields) {
         using namespace ::google::protobuf;
         const Descriptor& descriptor = *protoTo.GetDescriptor();
         const Reflection& reflectionTo = *protoTo.GetReflection();
@@ -490,24 +490,6 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
         }
     }
 
-    template<typename TMessage>
-    static ::google::protobuf::RepeatedField<arc_i32> GetDefaultFields(const TMessage& message) {
-        using namespace ::google::protobuf;
-        const Descriptor& descriptor = *message.GetDescriptor();
-        ::google::protobuf::RepeatedField<arc_i32> defaultFields;
-        int fieldCount = descriptor.field_count();
-        for (int index = 0; index < fieldCount; ++index) {
-            const FieldDescriptor* field = descriptor.field(index);
-            const auto& options(field->options());
-            if (options.HasExtension(NKikimrWhiteboard::DefaultField)) {
-                if (options.GetExtension(NKikimrWhiteboard::DefaultField)) {
-                    defaultFields.Add(field->number());
-                }
-            }
-        }
-        return defaultFields;
-    }
-
     template<typename TMessage, typename TRequest>
     static void Copy(TMessage& to, const TMessage& from, const TRequest& request) {
         if (request.FieldsRequiredSize() > 0) {
@@ -517,8 +499,7 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
                 SelectiveCopy(to, from, request.GetFieldsRequired());
             }
         } else {
-            static auto defaultFields = GetDefaultFields(to);
-            SelectiveCopy(to, from, defaultFields);
+            SelectiveCopy(to, from, GetDefaultWhiteboardFields<TMessage>());
         }
     }
 
@@ -1121,6 +1102,30 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
     }
 };
 
+template<typename TMessage>
+::google::protobuf::RepeatedField<int> InitDefaultWhiteboardFields() {
+    using namespace ::google::protobuf;
+    const Descriptor& descriptor = *TMessage::GetDescriptor();
+    ::google::protobuf::RepeatedField<int> defaultFields;
+    int fieldCount = descriptor.field_count();
+    for (int index = 0; index < fieldCount; ++index) {
+        const FieldDescriptor* field = descriptor.field(index);
+        const auto& options(field->options());
+        if (options.HasExtension(NKikimrWhiteboard::DefaultField)) {
+            if (options.GetExtension(NKikimrWhiteboard::DefaultField)) {
+                defaultFields.Add(field->number());
+            }
+        }
+    }
+    return defaultFields;
+}
+
+template<typename TMessage>
+::google::protobuf::RepeatedField<int> GetDefaultWhiteboardFields() {
+    static ::google::protobuf::RepeatedField<int> defaultFields = InitDefaultWhiteboardFields<TMessage>();
+    return defaultFields;
+}
+
 IActor* CreateNodeWhiteboardService() {
     return new TNodeWhiteboardService();
 }

ydb/core/viewer/json_handlers_viewer.cpp

@@ -203,7 +203,7 @@ void InitViewerStorageUsageJsonHandler(TJsonHandlers &handlers) {
 }
 
 void InitViewerClusterJsonHandler(TJsonHandlers& handlers) {
-    handlers.AddHandler("/viewer/cluster", new TJsonHandler<TJsonCluster>(TJsonCluster::GetSwagger()), 4);
+    handlers.AddHandler("/viewer/cluster", new TJsonHandler<TJsonCluster>(TJsonCluster::GetSwagger()), 5);
 }
 
 void InitViewerLabeledCountersJsonHandler(TJsonHandlers &handlers) {
@@ -219,15 +219,15 @@ void InitViewerHiveStatsJsonHandler(TJsonHandlers& handlers) {
 }
 
 void InitViewerTenantInfoJsonHandler(TJsonHandlers &handlers) {
-    handlers.AddHandler("/viewer/tenantinfo", new TJsonHandler<TJsonTenantInfo>(TJsonTenantInfo::GetSwagger()), 2);
+    handlers.AddHandler("/viewer/tenantinfo", new TJsonHandler<TJsonTenantInfo>(TJsonTenantInfo::GetSwagger()), 3);
 }
 
 void InitViewerWhoAmIJsonHandler(TJsonHandlers& handlers) {
     handlers.AddHandler("/viewer/whoami", new TJsonHandler<TJsonWhoAmI>(TJsonWhoAmI::GetSwagger()));
 }
 
 void InitViewerQueryJsonHandler(TJsonHandlers& handlers) {
-    handlers.AddHandler("/viewer/query", new TJsonHandler<TJsonQuery>(TJsonQuery::GetSwagger()), 3);
+    handlers.AddHandler("/viewer/query", new TJsonHandler<TJsonQuery>(TJsonQuery::GetSwagger()), 4);
 }
 
 void InitViewerNetInfoJsonHandler(TJsonHandlers& handlers) {
@@ -243,7 +243,7 @@ void InitViewerHealthCheckJsonHandler(TJsonHandlers& handlers) {
 }
 
 void InitViewerNodesJsonHandler(TJsonHandlers& handlers) {
-    handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 9);
+    handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 11);
 }
 
 void InitViewerACLJsonHandler(TJsonHandlers &jsonHandlers) {

ydb/core/viewer/json_local_rpc.h

@@ -128,10 +128,9 @@ class TJsonLocalRpc : public TViewerPipeClient {
                 ReplyAndPassAway(GetHTTPBADREQUEST("text/plain", e.what()));
                 return false;
             }
-        } else {
-            const auto& params(Event->Get()->Request.GetParams());
-            Params2Proto(params, request);
         }
+        const auto& params(Event->Get()->Request.GetParams());
+        Params2Proto(params, request);
         if (!ValidateRequest(request)) {
             return false;
         }

ydb/core/viewer/json_pipe_req.cpp

@@ -1,4 +1,5 @@
 #include "json_pipe_req.h"
+#include <library/cpp/json/json_reader.h>
 #include <library/cpp/json/json_writer.h>
 
 namespace NKikimr::NViewer {
@@ -25,7 +26,28 @@ TViewerPipeClient::TViewerPipeClient(IViewer* viewer, NMon::TEvHttpInfo::TPtr& e
     : Viewer(viewer)
     , Event(ev)
 {
-    InitConfig(Event->Get()->Request.GetParams());
+    TCgiParameters params = Event->Get()->Request.GetParams();
+    if (Event->Get()->Request.GetHeader("Content-Type") == "application/json") {
+        NJson::TJsonValue jsonData;
+        if (NJson::ReadJsonTree(Event->Get()->Request.GetPostContent(), &jsonData)) {
+            if (jsonData.IsMap()) {
+                for (const auto& [key, value] : jsonData.GetMap()) {
+                    switch (value.GetType()) {
+                        case NJson::EJsonValueType::JSON_STRING:
+                        case NJson::EJsonValueType::JSON_INTEGER:
+                        case NJson::EJsonValueType::JSON_UINTEGER:
+                        case NJson::EJsonValueType::JSON_DOUBLE:
+                        case NJson::EJsonValueType::JSON_BOOLEAN:
+                            params.InsertUnescaped(key, value.GetStringRobust());
+                            break;
+                        default:
+                            break;
+                    }
+                }
+            }
+        }
+    }
+    InitConfig(params);
     NWilson::TTraceId traceId;
     TStringBuf traceparent = Event->Get()->Request.GetHeader("traceparent");
     if (traceparent) {
@@ -645,10 +667,20 @@ TRequestState TViewerPipeClient::GetRequest() const {
 }
 
 void TViewerPipeClient::ReplyAndPassAway(TString data, const TString& error) {
+    TString message = error;
     Send(Event->Sender, new NMon::TEvHttpInfoRes(data, 0, NMon::IEvHttpInfoRes::EContentType::Custom));
+    if (message.empty()) {
+        TStringBuf dataParser(data);
+        if (dataParser.NextTok(' ') == "HTTP/1.1") {
+            TStringBuf code = dataParser.NextTok(' ');
+            if (code.size() == 3 && code[0] != '2') {
+                message = dataParser.NextTok('\n');
+            }
+        }
+    }
     if (Span) {
-        if (error) {
-            Span.EndError(error);
+        if (message) {
+            Span.EndError(message);
         } else {
             Span.EndOk();
         }

ydb/core/viewer/protos/viewer.proto

@@ -322,6 +322,7 @@ message TClusterInfo {
     uint32 NodesTotal = 10;
     uint32 NodesAlive = 11;
     uint32 NumberOfCpus = 20;
+    uint32 CoresTotal = 24;
     double CoresUsed = 21;
     double LoadAverage = 22;
     repeated NKikimrWhiteboard.TSystemStateInfo.TPoolStats PoolStats = 23;
@@ -358,7 +359,7 @@ message TStorageUsage {
         SSD = 2;
     }
     EType Type = 1;
-    uint64 Size = 2;
+    optional uint64 Size = 2;
     uint64 Limit = 3;
     uint64 SoftQuota = 4;
     uint64 HardQuota = 5;
@@ -394,6 +395,7 @@ message TTenant {
     Ydb.Cms.DatabaseQuotas DatabaseQuotas = 42;
     repeated TStorageUsage TablesStorage = 44;
     repeated TStorageUsage DatabaseStorage = 45;
+    uint32 CoresTotal = 50;
 }
 
 message TTenants {

ydb/core/viewer/storage_groups.h

@@ -151,7 +151,6 @@ class TStorageGroups : public TViewerPipeClient {
     std::unordered_set<TGroupId> FilterGroupIds;
     std::unordered_set<TNodeId> FilterNodeIds;
     std::unordered_set<ui32> FilterPDiskIds;
-    std::vector<TNodeId> SubscriptionNodeIds;
 
     enum class EWith {
         Everything,
@@ -894,22 +893,6 @@ class TStorageGroups : public TViewerPipeClient {
         Schedule(TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup(TimeoutFinal)); // timeout for the rest
     }
 
-    void PassAway() override {
-        std::vector<bool> passedNodes;
-        for (const TNodeId nodeId : SubscriptionNodeIds) {
-            if (passedNodes.size() <= nodeId) {
-                passedNodes.resize(nodeId + 1);
-            } else {
-                if (passedNodes[nodeId]) {
-                    continue;
-                }
-            }
-            Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe());
-            passedNodes[nodeId] = true;
-        }
-        TBase::PassAway();
-    }
-
     void ApplyFilter() {
         // database pre-filter, affects TotalGroups count
         if (!DatabaseStoragePools.empty()) {
@@ -1042,20 +1025,29 @@ class TStorageGroups : public TViewerPipeClient {
                 With = EWith::Everything;
                 GroupsByGroupId.clear();
             }
-            if (!Filter.empty() && FieldsAvailable.test(+EGroupFields::PoolName) && FieldsAvailable.test(+EGroupFields::GroupId)) {
-                TVector<TString> filterWords = SplitString(Filter, " ");
-                TGroupView groupView;
-                for (TGroup* group : GroupView) {
-                    for (const TString& word : filterWords) {
-                        if (group->PoolName.Contains(word) || ::ToString(group->GroupId).Contains(word)) {
-                            groupView.push_back(group);
-                            break;
+            if (!Filter.empty()) {
+                bool allFieldsPresent =
+                    (!FieldsRequired.test(+EGroupFields::GroupId) || FieldsAvailable.test(+EGroupFields::GroupId)) &&
+                    (!FieldsRequired.test(+EGroupFields::PoolName) || FieldsAvailable.test(+EGroupFields::PoolName));
+                if (allFieldsPresent) {
+                    TVector<TString> filterWords = SplitString(Filter, " ");
+                    TGroupView groupView;
+                    for (TGroup* group : GroupView) {
+                        for (const TString& word : filterWords) {
+                            if (FieldsRequired.test(+EGroupFields::GroupId) && ::ToString(group->GroupId).Contains(word)) {
+                                groupView.push_back(group);
+                                break;
+                            }
+                            if (FieldsRequired.test(+EGroupFields::PoolName) && group->PoolName.Contains(word)) {
+                                groupView.push_back(group);
+                                break;
+                            }
                         }
                     }
+                    GroupView.swap(groupView);
+                    Filter.clear();
+                    GroupsByGroupId.clear();
                 }
-                GroupView.swap(groupView);
-                Filter.clear();
-                GroupsByGroupId.clear();
             }
             if (!FilterGroup.empty() && FieldsAvailable.test(+FilterGroupBy)) {
                 TGroupView groupView;
@@ -1220,6 +1212,7 @@ class TStorageGroups : public TViewerPipeClient {
     bool CollectedHiveData = false;
 
     void CollectHiveData() {
+        static TPathId badPathId(0, 0);
         if (!CollectedHiveData) {
             if (!GroupView.empty()) {
                 ui64 hiveId = AppData()->DomainsInfo->GetHive();
@@ -1232,6 +1225,9 @@ class TStorageGroups : public TViewerPipeClient {
             }
             for (const TGroup* group : GroupView) {
                 TPathId pathId(group->SchemeShardId, group->PathId);
+                if (pathId == badPathId) {
+                    pathId = {AppData()->DomainsInfo->Domain->SchemeRoot, 1};
+                }
                 if (NavigateKeySetResult.count(pathId) == 0) {
                     ui64 cookie = NavigateKeySetResult.size();
                     NavigateKeySetResult.emplace(pathId, MakeRequestSchemeCacheNavigate(pathId, cookie));
@@ -1840,12 +1836,7 @@ class TStorageGroups : public TViewerPipeClient {
             return;
         }
         if (BSGroupStateResponse.count(nodeId) == 0) {
-            TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId);
-            BSGroupStateResponse.emplace(nodeId, MakeRequest<TEvWhiteboard::TEvBSGroupStateResponse>(whiteboardServiceId,
-                new TEvWhiteboard::TEvBSGroupStateRequest(),
-                IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
-                nodeId));
-            SubscriptionNodeIds.push_back(nodeId);
+            BSGroupStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvBSGroupStateRequest()));
             ++BSGroupStateRequestsInFlight;
         }
     }
@@ -1854,22 +1845,13 @@ class TStorageGroups : public TViewerPipeClient {
         if (nodeId == 0) {
             return;
         }
-        TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId);
         if (VDiskStateResponse.count(nodeId) == 0) {
-            VDiskStateResponse.emplace(nodeId, MakeRequest<TEvWhiteboard::TEvVDiskStateResponse>(whiteboardServiceId,
-                new TEvWhiteboard::TEvVDiskStateRequest(),
-                IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
-                nodeId));
+            VDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvVDiskStateRequest()));
             ++VDiskStateRequestsInFlight;
-            SubscriptionNodeIds.push_back(nodeId);
         }
         if (PDiskStateResponse.count(nodeId) == 0) {
-            PDiskStateResponse.emplace(nodeId, MakeRequest<TEvWhiteboard::TEvPDiskStateResponse>(whiteboardServiceId,
-                new TEvWhiteboard::TEvPDiskStateRequest(),
-                IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
-                nodeId));
+            PDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvPDiskStateRequest()));
             ++PDiskStateRequestsInFlight;
-            SubscriptionNodeIds.push_back(nodeId);
         }
     }
 

ydb/core/viewer/viewer_autocomplete.h

@@ -138,7 +138,6 @@ class TJsonAutocomplete : public TViewerPipeClient {
     TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult> MakeRequestSchemeCacheNavigate() {
         auto request = std::make_unique<NSchemeCache::TSchemeCacheNavigate>();
         for (const TString& path : Paths) {
-            Cerr << "Looking into " << path << Endl;
             NSchemeCache::TSchemeCacheNavigate::TEntry entry;
             entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList;
             entry.SyncVersion = false;