Skip to content

Commit 62becca

Browse files
GrigoriyPAGrigoriyPA
authored
Merge a16614d into 2876ba4 
2 parents 2876ba4 + a16614d commit 62becca

20 files changed

+519
-175
lines changed

‎ydb/core/driver_lib/run/config_helpers.cpp

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
#include "config_helpers.h"
22

3+
#include <ydb/core/base/localdb.h>
4+
#include <ydb/core/protos/bootstrap.pb.h>
5+
#include <ydb/core/protos/resource_broker.pb.h>
6+
37
#include <ydb/library/actors/util/affinity.h>
48

59

@@ -114,4 +118,31 @@ NActors::TSchedulerConfig CreateSchedulerConfig(const NKikimrConfig::TActorSyste
114118

115119
} // namespace NActorSystemConfigHelpers
116120

121+
namespace NKikimrConfigHelpers {
122+
123+
NMemory::TResourceBrokerConfig CreateMemoryControllerResourceBrokerConfig(const NKikimrConfig::TAppConfig& config) {
124+
NMemory::TResourceBrokerConfig resourceBrokerSelfConfig; // for backward compatibility
125+
auto mergeResourceBrokerConfigs = [&](const NKikimrResourceBroker::TResourceBrokerConfig& resourceBrokerConfig) {
126+
if (resourceBrokerConfig.HasResourceLimit() && resourceBrokerConfig.GetResourceLimit().HasMemory()) {
127+
resourceBrokerSelfConfig.LimitBytes = resourceBrokerConfig.GetResourceLimit().GetMemory();
128+
}
129+
for (const auto& queue : resourceBrokerConfig.GetQueues()) {
130+
if (queue.GetName() == NLocalDb::KqpResourceManagerQueue) {
131+
if (queue.HasLimit() && queue.GetLimit().HasMemory()) {
132+
resourceBrokerSelfConfig.QueryExecutionLimitBytes = queue.GetLimit().GetMemory();
133+
}
134+
}
135+
}
136+
};
137+
if (config.HasBootstrapConfig() && config.GetBootstrapConfig().HasResourceBroker()) {
138+
mergeResourceBrokerConfigs(config.GetBootstrapConfig().GetResourceBroker());
139+
}
140+
if (config.HasResourceBrokerConfig()) {
141+
mergeResourceBrokerConfigs(config.GetResourceBrokerConfig());
142+
}
143+
return resourceBrokerSelfConfig;
144+
}
145+
146+
} // namespace NKikimrConfigHelpers
147+
117148
} // namespace NKikimr

‎ydb/core/driver_lib/run/config_helpers.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include <ydb/core/memory_controller/memory_controller.h>
34
#include <ydb/core/protos/config.pb.h>
45

56
#include <ydb/library/actors/core/config.h>
@@ -15,4 +16,10 @@ NActors::TSchedulerConfig CreateSchedulerConfig(const NKikimrConfig::TActorSyste
1516

1617
} // namespace NActorSystemConfigHelpers
1718

19+
namespace NKikimrConfigHelpers {
20+
21+
NMemory::TResourceBrokerConfig CreateMemoryControllerResourceBrokerConfig(const NKikimrConfig::TAppConfig& config);
22+
23+
} // namespace NKikimrConfigHelpers
24+
1825
} // namespace NKikimr

‎ydb/core/driver_lib/run/kikimr_services_initializers.cpp

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2052,28 +2052,8 @@ void TMemoryControllerInitializer::InitializeServices(
20522052
NActors::TActorSystemSetup* setup,
20532053
const NKikimr::TAppData* appData)
20542054
{
2055-
NMemory::TResourceBrokerConfig resourceBrokerSelfConfig; // for backward compatibility
2056-
auto mergeResourceBrokerConfigs = [&](const NKikimrResourceBroker::TResourceBrokerConfig& resourceBrokerConfig) {
2057-
if (resourceBrokerConfig.HasResourceLimit() && resourceBrokerConfig.GetResourceLimit().HasMemory()) {
2058-
resourceBrokerSelfConfig.LimitBytes = resourceBrokerConfig.GetResourceLimit().GetMemory();
2059-
}
2060-
for (const auto& queue : resourceBrokerConfig.GetQueues()) {
2061-
if (queue.GetName() == NLocalDb::KqpResourceManagerQueue) {
2062-
if (queue.HasLimit() && queue.GetLimit().HasMemory()) {
2063-
resourceBrokerSelfConfig.QueryExecutionLimitBytes = queue.GetLimit().GetMemory();
2064-
}
2065-
}
2066-
}
2067-
};
2068-
if (Config.HasBootstrapConfig() && Config.GetBootstrapConfig().HasResourceBroker()) {
2069-
mergeResourceBrokerConfigs(Config.GetBootstrapConfig().GetResourceBroker());
2070-
}
2071-
if (Config.HasResourceBrokerConfig()) {
2072-
mergeResourceBrokerConfigs(Config.GetResourceBrokerConfig());
2073-
}
2074-
20752055
auto* actor = NMemory::CreateMemoryController(TDuration::Seconds(1), ProcessMemoryInfoProvider,
2076-
Config.GetMemoryControllerConfig(), resourceBrokerSelfConfig,
2056+
Config.GetMemoryControllerConfig(), NKikimrConfigHelpers::CreateMemoryControllerResourceBrokerConfig(Config),
20772057
appData->Counters);
20782058
setup->LocalServices.emplace_back(
20792059
NMemory::MakeMemoryControllerId(0),

‎ydb/core/testlib/basics/helpers.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ namespace NFake {
1818
ui64 SectorSize = 0;
1919
ui64 ChunkSize = 0;
2020
ui64 DiskSize = 0;
21+
bool FormatDisk = true;
22+
TString DiskPath;
2123
};
2224

2325
struct INode {

‎ydb/core/testlib/basics/storage.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,15 @@ namespace NKikimr {
5454

5555
static ui64 keySalt = 0;
5656
ui64 salt = ++keySalt;
57-
TString baseDir = Runtime.GetTempDir();
57+
TString baseDir = conf.DiskPath ? conf.DiskPath : Runtime.GetTempDir();
5858

5959
if (Conf.UseDisk) {
6060
MakeDirIfNotExist(baseDir.c_str());
6161
}
6262

6363
PDiskPath = TStringBuilder() << baseDir << "pdisk_1.dat";
6464

65-
if (!Mock) {
65+
if (!Mock && conf.FormatDisk) {
6666
FormatPDisk(PDiskPath,
6767
Conf.DiskSize, Conf.SectorSize, Conf.ChunkSize, PDiskGuid,
6868
0x123 + salt, 0x456 + salt, 0x789 + salt, mainKey,

‎ydb/core/testlib/test_client.cpp

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@
9393
#include <ydb/core/mind/tenant_slot_broker.h>
9494
#include <ydb/core/mind/tenant_node_enumeration.h>
9595
#include <ydb/core/mind/node_broker.h>
96+
#include <ydb/core/mon_alloc/monitor.h>
9697
#include <ydb/core/kesus/tablet/events.h>
9798
#include <ydb/core/sys_view/service/sysview_service.h>
9899
#include <yql/essentials/minikql/mkql_function_registry.h>
@@ -613,18 +614,21 @@ namespace Tests {
613614

614615
NKikimrBlobStorage::TDefineBox boxConfig;
615616
boxConfig.SetBoxId(Settings->BOX_ID);
617+
boxConfig.SetItemConfigGeneration(Settings->StorageGeneration);
616618

617619
ui32 nodeId = Runtime->GetNodeId(0);
618620
Y_ABORT_UNLESS(nodesInfo->Nodes[0].NodeId == nodeId);
619621
auto& nodeInfo = nodesInfo->Nodes[0];
620622

621623
NKikimrBlobStorage::TDefineHostConfig hostConfig;
622624
hostConfig.SetHostConfigId(nodeId);
625+
hostConfig.SetItemConfigGeneration(Settings->StorageGeneration);
623626
TString path;
624627
if (Settings->UseSectorMap) {
625628
path ="SectorMap:test-client[:2000]";
626629
} else {
627-
path = TStringBuilder() << Runtime->GetTempDir() << "pdisk_1.dat";
630+
TString diskPath = Settings->CustomDiskParams.DiskPath;
631+
path = TStringBuilder() << (diskPath ? diskPath : Runtime->GetTempDir()) << "pdisk_1.dat";
628632
}
629633
hostConfig.AddDrive()->SetPath(path);
630634
if (Settings->Verbose) {
@@ -640,7 +644,9 @@ namespace Tests {
640644

641645
for (const auto& [poolKind, storagePool] : Settings->StoragePoolTypes) {
642646
if (storagePool.GetNumGroups() > 0) {
643-
bsConfigureRequest->Record.MutableRequest()->AddCommand()->MutableDefineStoragePool()->CopyFrom(storagePool);
647+
auto* command = bsConfigureRequest->Record.MutableRequest()->AddCommand()->MutableDefineStoragePool();
648+
command->CopyFrom(storagePool);
649+
command->SetItemConfigGeneration(Settings->StorageGeneration);
644650
}
645651
}
646652

@@ -1074,6 +1080,28 @@ namespace Tests {
10741080
}
10751081
}
10761082

1083+
{
1084+
if (Settings->NeedStatsCollectors) {
1085+
TString filePathPrefix;
1086+
if (Settings->AppConfig->HasMonitoringConfig()) {
1087+
filePathPrefix = Settings->AppConfig->GetMonitoringConfig().GetMemAllocDumpPathPrefix();
1088+
}
1089+
1090+
const TIntrusivePtr<NMemory::IProcessMemoryInfoProvider> processMemoryInfoProvider(MakeIntrusive<NMemory::TProcessMemoryInfoProvider>());
1091+
1092+
IActor* monitorActor = CreateMemProfMonitor(TDuration::Seconds(1), processMemoryInfoProvider,
1093+
Runtime->GetAppData(nodeIdx).Counters, filePathPrefix);
1094+
const TActorId monitorActorId = Runtime->Register(monitorActor, nodeIdx, Runtime->GetAppData(nodeIdx).BatchPoolId);
1095+
Runtime->RegisterService(MakeMemProfMonitorID(Runtime->GetNodeId(nodeIdx)), monitorActorId, nodeIdx);
1096+
1097+
IActor* controllerActor = NMemory::CreateMemoryController(TDuration::Seconds(1), processMemoryInfoProvider,
1098+
Settings->AppConfig->GetMemoryControllerConfig(), NKikimrConfigHelpers::CreateMemoryControllerResourceBrokerConfig(*Settings->AppConfig),
1099+
Runtime->GetAppData(nodeIdx).Counters);
1100+
const TActorId controllerActorId = Runtime->Register(controllerActor, nodeIdx, Runtime->GetAppData(nodeIdx).BatchPoolId);
1101+
Runtime->RegisterService(NMemory::MakeMemoryControllerId(0), controllerActorId, nodeIdx);
1102+
}
1103+
}
1104+
10771105
{
10781106
IActor* kesusService = NKesus::CreateKesusProxyService();
10791107
TActorId kesusServiceId = Runtime->Register(kesusService, nodeIdx, userPoolId);

‎ydb/core/testlib/test_client.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ namespace Tests {
124124
TString DomainName = TestDomainName;
125125
ui32 NodeCount = 1;
126126
ui32 DynamicNodeCount = 0;
127+
ui64 StorageGeneration = 0;
127128
NFake::TStorage CustomDiskParams;
128129
TControls Controls;
129130
TAppPrepare::TFnReg FrFactory = &DefaultFrFactory;
@@ -179,6 +180,7 @@ namespace Tests {
179180
TServerSettings& SetDomainName(const TString& value);
180181
TServerSettings& SetNodeCount(ui32 value) { NodeCount = value; return *this; }
181182
TServerSettings& SetDynamicNodeCount(ui32 value) { DynamicNodeCount = value; return *this; }
183+
TServerSettings& SetStorageGeneration(ui64 value) { StorageGeneration = value; return *this; }
182184
TServerSettings& SetCustomDiskParams(const NFake::TStorage& value) { CustomDiskParams = value; return *this; }
183185
TServerSettings& SetControls(const TControls& value) { Controls = value; return *this; }
184186
TServerSettings& SetFrFactory(const TAppPrepare::TFnReg& value) { FrFactory = value; return *this; }

‎ydb/tests/tools/kqprun/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
storage
12
sync_dir
23
example
34
udfs
5+
46
*.log
57
*.json
68
*.sql

‎ydb/tests/tools/kqprun/flame_graph.sh

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
#!/usr/bin/env bash
22

3-
# For svg graph download https://github.com/brendangregg/FlameGraph
4-
# and run `FlameGraph/stackcollapse-perf.pl profdata.txt | FlameGraph/flamegraph.pl > profdata.svg`
3+
set -eux
54

6-
pid=$(pgrep -u $USER kqprun)
5+
kqprun_pid=$(pgrep -u $USER kqprun)
76

8-
echo "Target process id: ${pid}"
9-
10-
sudo perf record -F 50 --call-graph dwarf -g --proc-map-timeout=10000 --pid $pid -v -o profdata -- sleep 30
7+
sudo perf record -F 50 --call-graph dwarf -g --proc-map-timeout=10000 --pid $kqprun_pid -v -o profdata -- sleep ${1:-'30'}
118
sudo perf script -i profdata > profdata.txt
9+
10+
flame_graph_tool="../../../../contrib/tools/flame-graph/"
11+
12+
${flame_graph_tool}/stackcollapse-perf.pl profdata.txt | ${flame_graph_tool}/flamegraph.pl > profdata.svg

0 commit comments

Comments
 (0)