add subdomain key to required fiels for non-database requests (#15341)

Author: adameat

ydb/core/viewer/json_handlers_viewer.cpp

@@ -246,7 +246,7 @@ void InitViewerHealthCheckJsonHandler(TJsonHandlers& handlers) {
 }
 
 void InitViewerNodesJsonHandler(TJsonHandlers& handlers) {
-    handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 14);
+    handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 15);
 }
 
 void InitViewerACLJsonHandler(TJsonHandlers &jsonHandlers) {

ydb/core/viewer/protos/viewer.proto

@@ -561,6 +561,12 @@ message TNodeInfo {
     repeated NKikimrWhiteboard.TNodeStateInfo ReversePeers = 55;
 }
 
+message TNodeBatch {
+    repeated uint32 NodesToAskFor = 1;
+    repeated uint32 NodesToAskAbout = 2;
+    optional bool HasStaticNodes = 3;
+}
+
 message TNodesInfo {
     uint32 Version = 1;
     optional uint64 TotalNodes = 2;
@@ -578,6 +584,7 @@ message TNodesInfo {
     optional uint64 MaximumSlotsPerDisk = 51;
     optional bool NoDC = 60;
     optional bool NoRack = 61;
+    repeated TNodeBatch OriginalNodeBatches = 62;
 }
 
 enum ENodeType {

ydb/core/viewer/tests/canondata/result.json

@@ -2969,7 +2969,7 @@
     "test.test_viewer_nodes_issue_14992": {
         "response_group": {
             "FieldsAvailable": "0000000110111110111111100000111",
-            "FieldsRequired": "0000000001000000010000000000101",
+            "FieldsRequired": "0000000001100000010000000000101",
             "FoundNodes": "3",
             "MaximumDisksPerNode": "1",
             "Nodes": [
@@ -3190,7 +3190,7 @@
         },
         "response_group_by": {
             "FieldsAvailable": "0000000110111110111111100000111",
-            "FieldsRequired": "0000000001000000010000000000101",
+            "FieldsRequired": "0000000001100000010000000000101",
             "FoundNodes": "3",
             "MaximumDisksPerNode": "1",
             "NodeGroups": [

ydb/core/viewer/viewer_nodes.h

@@ -181,6 +181,7 @@ class TJsonNodes : public TViewerPipeClient {
     ui32 SpaceUsageProblem = 90; // %
     bool OffloadMerge = true;
     size_t OffloadMergeAttempts = 2;
+    size_t OffloadMergeBatchSize = 200;
 
     using TGroupSortKey = std::variant<TString, ui64, float, int>;
 
@@ -748,6 +749,8 @@ class TJsonNodes : public TViewerPipeClient {
     std::vector<TNodeGroup> NodeGroups;
     std::unordered_map<TNodeId, TNode*> NodesByNodeId;
     std::unordered_map<TNodeId, TNodeBatch> NodeBatches;
+    std::vector<TNodeBatch> OriginalNodeBatches;
+    bool DumpOriginalNodeBatches = false;
 
     TFieldsType FieldsRequired;
     TFieldsType FieldsAvailable;
@@ -953,8 +956,10 @@ class TJsonNodes : public TViewerPipeClient {
             }
         }
 
-        OffloadMerge = FromStringWithDefault<bool>(params.Get("offload_merge"), OffloadMerge);
-        OffloadMergeAttempts = FromStringWithDefault<bool>(params.Get("offload_merge_attempts"), OffloadMergeAttempts);
+        OffloadMerge = FromStringWithDefault(params.Get("offload_merge"), OffloadMerge);
+        OffloadMergeAttempts = FromStringWithDefault(params.Get("offload_merge_attempts"), OffloadMergeAttempts);
+        OffloadMergeBatchSize = FromStringWithDefault(params.Get("offload_merge_batch_size"), OffloadMergeBatchSize);
+        DumpOriginalNodeBatches = FromStringWithDefault(params.Get("dump_original_node_batches"), DumpOriginalNodeBatches);
         Direct = FromStringWithDefault<bool>(params.Get("direct"), Direct);
         TString filterStoragePool = params.Get("pool");
         if (filterStoragePool.empty()) {
@@ -1078,6 +1083,9 @@ class TJsonNodes : public TViewerPipeClient {
             request->Record.AddFieldsRequired(-1);
             NodeStateResponse = MakeWhiteboardRequest(TActivationContext::ActorSystem()->NodeId, request.release());
         }
+        if (!FilterDatabase && OffloadMerge && FieldsNeeded(FieldsSystemState)) {
+            FieldsRequired.set(+ENodeFields::SubDomainKey);
+        }
         if (!FilterStoragePools.empty() || !FilterGroupIds.empty()) {
             FilterDatabase = false; // we disable database filter if we're filtering by pool or group
         }
@@ -1571,8 +1579,6 @@ class TJsonNodes : public TViewerPipeClient {
         ApplyLimit();
     }
 
-    static constexpr size_t BATCH_SIZE = 200;
-
     void BuildCandidates(TNodeBatch& batch, std::vector<TNode*>& candidates) {
         auto itCandidate = candidates.begin();
         for (; itCandidate != candidates.end() && batch.NodesToAskFor.size() < OffloadMergeAttempts; ++itCandidate) {
@@ -1591,9 +1597,9 @@ class TJsonNodes : public TViewerPipeClient {
         std::sort(candidates.begin(), candidates.end(), [](TNode* a, TNode* b) {
             return a->GetCandidateScore() > b->GetCandidateScore();
         });
-        while (nodeBatch.NodesToAskAbout.size() > BATCH_SIZE) {
+        while (nodeBatch.NodesToAskAbout.size() > OffloadMergeBatchSize) {
             TNodeBatch newBatch;
-            size_t splitSize = std::min(BATCH_SIZE, nodeBatch.NodesToAskAbout.size() / 2);
+            size_t splitSize = std::min(OffloadMergeBatchSize, nodeBatch.NodesToAskAbout.size() / 2);
             newBatch.NodesToAskAbout.reserve(splitSize);
             for (size_t i = 0; i < splitSize; ++i) {
                 newBatch.NodesToAskAbout.push_back(nodeBatch.NodesToAskAbout.back());
@@ -2105,6 +2111,9 @@ class TJsonNodes : public TViewerPipeClient {
                 FieldsRequired.set(+ENodeFields::SystemState);
             }
             std::vector<TNodeBatch> batches = BatchNodes(NodeView);
+            if (DumpOriginalNodeBatches) {
+                OriginalNodeBatches = batches;
+            }
             if (FilterPeerRole == EPeerRole::Static || FilterPeerRole == EPeerRole::Other) {
                 for (TNodeBatch& batch : batches) {
                     batch.FieldsRequested.set(+ENodeFields::Peers);
@@ -3101,6 +3110,16 @@ class TJsonNodes : public TViewerPipeClient {
         AddEvent("ReplyAndPassAway");
         ApplyEverything();
         NKikimrViewer::TNodesInfo json;
+        for (const auto& batch : OriginalNodeBatches) {
+            auto* jsonBatch = json.AddOriginalNodeBatches();
+            for (const auto& node : batch.NodesToAskFor) {
+                jsonBatch->AddNodesToAskFor(node->GetNodeId());
+            }
+            for (const auto& node : batch.NodesToAskAbout) {
+                jsonBatch->AddNodesToAskAbout(node->GetNodeId());
+            }
+            jsonBatch->SetHasStaticNodes(batch.HasStaticNodes);
+        }
         json.SetVersion(Viewer->GetCapabilityVersion("/viewer/nodes"));
         json.SetFieldsAvailable(FieldsAvailable.to_string());
         json.SetFieldsRequired(FieldsRequired.to_string());