diff --git a/ydb/core/viewer/json_handlers_viewer.cpp b/ydb/core/viewer/json_handlers_viewer.cpp index aa39185ef068..b53294538da1 100644 --- a/ydb/core/viewer/json_handlers_viewer.cpp +++ b/ydb/core/viewer/json_handlers_viewer.cpp @@ -246,7 +246,7 @@ void InitViewerHealthCheckJsonHandler(TJsonHandlers& handlers) { } void InitViewerNodesJsonHandler(TJsonHandlers& handlers) { - handlers.AddHandler("/viewer/nodes", new TJsonHandler(TJsonNodes::GetSwagger()), 14); + handlers.AddHandler("/viewer/nodes", new TJsonHandler(TJsonNodes::GetSwagger()), 15); } void InitViewerACLJsonHandler(TJsonHandlers &jsonHandlers) { diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index dff4b2d4cbc2..e878aa227bd0 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -561,6 +561,12 @@ message TNodeInfo { repeated NKikimrWhiteboard.TNodeStateInfo ReversePeers = 55; } +message TNodeBatch { + repeated uint32 NodesToAskFor = 1; + repeated uint32 NodesToAskAbout = 2; + optional bool HasStaticNodes = 3; +} + message TNodesInfo { uint32 Version = 1; optional uint64 TotalNodes = 2; @@ -578,6 +584,7 @@ message TNodesInfo { optional uint64 MaximumSlotsPerDisk = 51; optional bool NoDC = 60; optional bool NoRack = 61; + repeated TNodeBatch OriginalNodeBatches = 62; } enum ENodeType { diff --git a/ydb/core/viewer/tests/canondata/result.json b/ydb/core/viewer/tests/canondata/result.json index f0859e110725..2a5258215a77 100644 --- a/ydb/core/viewer/tests/canondata/result.json +++ b/ydb/core/viewer/tests/canondata/result.json @@ -2975,7 +2975,7 @@ "test.test_viewer_nodes_issue_14992": { "response_group": { "FieldsAvailable": "0000000110111110111111100000111", - "FieldsRequired": "0000000001000000010000000000101", + "FieldsRequired": "0000000001100000010000000000101", "FoundNodes": "3", "MaximumDisksPerNode": "1", "Nodes": [ @@ -3196,7 +3196,7 @@ }, "response_group_by": { "FieldsAvailable": "0000000110111110111111100000111", - "FieldsRequired": "0000000001000000010000000000101", + "FieldsRequired": "0000000001100000010000000000101", "FoundNodes": "3", "MaximumDisksPerNode": "1", "NodeGroups": [ diff --git a/ydb/core/viewer/viewer_nodes.h b/ydb/core/viewer/viewer_nodes.h index afb35a9ec61e..affed74cddd7 100644 --- a/ydb/core/viewer/viewer_nodes.h +++ b/ydb/core/viewer/viewer_nodes.h @@ -181,6 +181,7 @@ class TJsonNodes : public TViewerPipeClient { ui32 SpaceUsageProblem = 90; // % bool OffloadMerge = true; size_t OffloadMergeAttempts = 2; + size_t OffloadMergeBatchSize = 200; using TGroupSortKey = std::variant; @@ -748,6 +749,8 @@ class TJsonNodes : public TViewerPipeClient { std::vector NodeGroups; std::unordered_map NodesByNodeId; std::unordered_map NodeBatches; + std::vector OriginalNodeBatches; + bool DumpOriginalNodeBatches = false; TFieldsType FieldsRequired; TFieldsType FieldsAvailable; @@ -953,8 +956,10 @@ class TJsonNodes : public TViewerPipeClient { } } - OffloadMerge = FromStringWithDefault(params.Get("offload_merge"), OffloadMerge); - OffloadMergeAttempts = FromStringWithDefault(params.Get("offload_merge_attempts"), OffloadMergeAttempts); + OffloadMerge = FromStringWithDefault(params.Get("offload_merge"), OffloadMerge); + OffloadMergeAttempts = FromStringWithDefault(params.Get("offload_merge_attempts"), OffloadMergeAttempts); + OffloadMergeBatchSize = FromStringWithDefault(params.Get("offload_merge_batch_size"), OffloadMergeBatchSize); + DumpOriginalNodeBatches = FromStringWithDefault(params.Get("dump_original_node_batches"), DumpOriginalNodeBatches); Direct = FromStringWithDefault(params.Get("direct"), Direct); TString filterStoragePool = params.Get("pool"); if (filterStoragePool.empty()) { @@ -1078,6 +1083,9 @@ class TJsonNodes : public TViewerPipeClient { request->Record.AddFieldsRequired(-1); NodeStateResponse = MakeWhiteboardRequest(TActivationContext::ActorSystem()->NodeId, request.release()); } + if (!FilterDatabase && OffloadMerge && FieldsNeeded(FieldsSystemState)) { + FieldsRequired.set(+ENodeFields::SubDomainKey); + } if (!FilterStoragePools.empty() || !FilterGroupIds.empty()) { FilterDatabase = false; // we disable database filter if we're filtering by pool or group } @@ -1571,8 +1579,6 @@ class TJsonNodes : public TViewerPipeClient { ApplyLimit(); } - static constexpr size_t BATCH_SIZE = 200; - void BuildCandidates(TNodeBatch& batch, std::vector& candidates) { auto itCandidate = candidates.begin(); for (; itCandidate != candidates.end() && batch.NodesToAskFor.size() < OffloadMergeAttempts; ++itCandidate) { @@ -1591,9 +1597,9 @@ class TJsonNodes : public TViewerPipeClient { std::sort(candidates.begin(), candidates.end(), [](TNode* a, TNode* b) { return a->GetCandidateScore() > b->GetCandidateScore(); }); - while (nodeBatch.NodesToAskAbout.size() > BATCH_SIZE) { + while (nodeBatch.NodesToAskAbout.size() > OffloadMergeBatchSize) { TNodeBatch newBatch; - size_t splitSize = std::min(BATCH_SIZE, nodeBatch.NodesToAskAbout.size() / 2); + size_t splitSize = std::min(OffloadMergeBatchSize, nodeBatch.NodesToAskAbout.size() / 2); newBatch.NodesToAskAbout.reserve(splitSize); for (size_t i = 0; i < splitSize; ++i) { newBatch.NodesToAskAbout.push_back(nodeBatch.NodesToAskAbout.back()); @@ -2105,6 +2111,9 @@ class TJsonNodes : public TViewerPipeClient { FieldsRequired.set(+ENodeFields::SystemState); } std::vector batches = BatchNodes(NodeView); + if (DumpOriginalNodeBatches) { + OriginalNodeBatches = batches; + } if (FilterPeerRole == EPeerRole::Static || FilterPeerRole == EPeerRole::Other) { for (TNodeBatch& batch : batches) { batch.FieldsRequested.set(+ENodeFields::Peers); @@ -3101,6 +3110,16 @@ class TJsonNodes : public TViewerPipeClient { AddEvent("ReplyAndPassAway"); ApplyEverything(); NKikimrViewer::TNodesInfo json; + for (const auto& batch : OriginalNodeBatches) { + auto* jsonBatch = json.AddOriginalNodeBatches(); + for (const auto& node : batch.NodesToAskFor) { + jsonBatch->AddNodesToAskFor(node->GetNodeId()); + } + for (const auto& node : batch.NodesToAskAbout) { + jsonBatch->AddNodesToAskAbout(node->GetNodeId()); + } + jsonBatch->SetHasStaticNodes(batch.HasStaticNodes); + } json.SetVersion(Viewer->GetCapabilityVersion("/viewer/nodes")); json.SetFieldsAvailable(FieldsAvailable.to_string()); json.SetFieldsRequired(FieldsRequired.to_string());