Skip to content

Commit d39295a

Browse files
alexvrualexvru
authored
Straighten BSC <-> NW interoperation protocol (merge from main) (#3575)
1 parent 295aafd commit d39295a

16 files changed

+261
-268
lines changed

‎ydb/core/base/blobstorage.h

+29-58
Original file line numberDiff line numberDiff line change
@@ -786,64 +786,35 @@ struct TEvBlobStorage {
786786
EvBunchOfEvents,
787787

788788
// blobstorage controller interface
789-
// EvControllerReadSchemeString = EvPut + 11 * 512,
790-
// EvControllerReadDataString,
791-
EvControllerRegisterNode = EvPut + 11 * 512 + 2,
792-
EvControllerCreatePDisk,
793-
EvControllerCreateVDiskSlots,
794-
EvControllerCreateGroup,
795-
EvControllerSelectGroups,
796-
EvControllerGetGroup,
797-
EvControllerUpdateDiskStatus,
798-
EvControllerUpdateGroupsUsage, // Not used.
799-
EvControllerConfigRequest,
800-
EvControllerConfigResponse,
801-
EvControllerProposeRequest,
802-
EvControllerProposeResponse,
803-
EvControllerVDiskStatusSubscribeRequest,
804-
EvControllerVDiskStatusReport,
805-
EvControllerGroupStatusRequest,
806-
EvControllerGroupStatusResponse,
807-
EvControllerUpdateGroup,
808-
EvControllerUpdateFaultyDisks,
809-
EvControllerProposeGroupKey,
810-
EvControllerUpdateGroupLatencies, // Not used.
811-
EvControllerUpdateGroupStat,
812-
EvControllerNotifyGroupChange,
813-
EvControllerCommitGroupLatencies,
814-
EvControllerUpdateSelfHealInfo,
815-
EvControllerScrubQueryStartQuantum,
816-
EvControllerScrubQuantumFinished,
817-
EvControllerScrubReportQuantumInProgress,
818-
EvControllerUpdateNodeDrives,
819-
EvControllerGroupDecommittedNotify,
820-
EvControllerGroupDecommittedResponse,
821-
EvControllerGroupMetricsExchange,
822-
823-
// EvControllerReadSchemeStringResult = EvPut + 12 * 512,
824-
// EvControllerReadDataStringResult,
825-
EvControllerNodeServiceSetUpdate = EvPut + 12 * 512 + 2,
826-
EvControllerCreatePDiskResult,
827-
EvControllerCreateVDiskSlotsResult,
828-
EvControllerCreateGroupResult,
829-
EvControllerSelectGroupsResult,
830-
EvRequestControllerInfo,
831-
EvResponseControllerInfo,
832-
EvControllerGroupReconfigureReplace, // Not used.
833-
EvControllerGroupReconfigureReplaceResult, // Not used.
834-
EvControllerGroupReconfigureWipe,
835-
EvControllerGroupReconfigureWipeResult,
836-
EvControllerNodeReport,
837-
EvControllerScrubStartQuantum,
838-
839-
EvControllerMigrationPause,
840-
EvControllerMigrationContinue,
841-
EvControllerMigrationFinished,
842-
EvControllerMigrationBatch,
843-
EvControllerMigrationBatchRequest,
844-
EvControllerMigrationDone,
845-
846-
EvControllerUpdateSystemViews,
789+
EvControllerRegisterNode = 0x10031602,
790+
EvControllerSelectGroups = 0x10031606,
791+
EvControllerGetGroup = 0x10031607,
792+
EvControllerUpdateDiskStatus = 0x10031608,
793+
EvControllerConfigRequest = 0x1003160a,
794+
EvControllerConfigResponse = 0x1003160b,
795+
EvControllerProposeGroupKey = 0x10031614,
796+
EvControllerUpdateGroupStat = 0x10031616,
797+
EvControllerNotifyGroupChange = 0x10031617,
798+
EvControllerCommitGroupLatencies = 0x10031618,
799+
EvControllerUpdateSelfHealInfo = 0x10031619,
800+
EvControllerScrubQueryStartQuantum = 0x1003161a,
801+
EvControllerScrubQuantumFinished = 0x1003161b,
802+
EvControllerScrubReportQuantumInProgress = 0x1003161c,
803+
EvControllerUpdateNodeDrives = 0x1003161d,
804+
EvControllerGroupDecommittedNotify = 0x1003161e,
805+
EvControllerGroupDecommittedResponse = 0x1003161f,
806+
EvControllerGroupMetricsExchange = 0x10031620,
807+
808+
// BSC interface result section
809+
EvControllerNodeServiceSetUpdate = 0x10031802,
810+
EvControllerSelectGroupsResult = 0x10031806,
811+
EvRequestControllerInfo = 0x10031807,
812+
EvResponseControllerInfo = 0x10031808,
813+
EvControllerGroupReconfigureWipe = 0x1003180b,
814+
EvControllerGroupReconfigureWipeResult = 0x1003180c,
815+
EvControllerNodeReport = 0x1003180d,
816+
EvControllerScrubStartQuantum = 0x1003180e,
817+
EvControllerUpdateSystemViews = 0x10031815,
847818

848819
// proxy - node controller interface
849820
EvConfigureProxy = EvPut + 13 * 512,

‎ydb/core/blobstorage/ut_blobstorage/balancing.cpp

+4-2
Original file line numberDiff line numberDiff line change
@@ -303,9 +303,11 @@ struct TRandomTest {
303303
ui32 pos = random() % Env->Settings.NodeCount;
304304
if (Env.RunningNodes.contains(pos)) {
305305
auto baseConfig = Env->FetchBaseConfig();
306-
const auto& somePDisk = baseConfig.GetPDisk(pos);
307306
const auto& someVSlot = baseConfig.GetVSlot(pos);
308-
Env->Wipe(somePDisk.GetNodeId(), somePDisk.GetPDiskId(), someVSlot.GetVSlotId().GetVSlotId());
307+
const auto& loc = someVSlot.GetVSlotId();
308+
Env->Wipe(loc.GetNodeId(), loc.GetPDiskId(), loc.GetVSlotId(),
309+
TVDiskID(someVSlot.GetGroupId(), someVSlot.GetGroupGeneration(), someVSlot.GetFailRealmIdx(),
310+
someVSlot.GetFailDomainIdx(), someVSlot.GetVDiskIdx()));
309311
Env->Sim(TDuration::Seconds(10));
310312
}
311313
}

‎ydb/core/blobstorage/ut_blobstorage/group_reconfiguration.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ Y_UNIT_TEST_SUITE(GroupReconfiguration) {
331331
Timer.Reset();
332332
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGetGroup>();
333333
ev->Record.AddGroupIDs(GroupId);
334-
NTabletPipe::SendData(SelfId(), ClientId, ev.release(), 0);
334+
NTabletPipe::SendData(SelfId(), ClientId, ev.release(), Max<ui64>());
335335
}
336336
}
337337

‎ydb/core/blobstorage/ut_blobstorage/lib/env.h

+11-8
Original file line numberDiff line numberDiff line change
@@ -751,17 +751,20 @@ struct TEnvironmentSetup {
751751
Sim(TDuration::Seconds(15));
752752
}
753753

754-
void Wipe(ui32 nodeId, ui32 pdiskId, ui32 vslotId) {
755-
const TActorId self = Runtime->AllocateEdgeActor(Settings.ControllerNodeId, __FILE__, __LINE__);
756-
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGroupReconfigureWipe>();
757-
auto& record = ev->Record;
758-
auto *vslot = record.MutableVSlotId();
754+
void Wipe(ui32 nodeId, ui32 pdiskId, ui32 vslotId, const TVDiskID& vdiskId) {
755+
NKikimrBlobStorage::TConfigRequest request;
756+
request.SetIgnoreGroupFailModelChecks(true);
757+
request.SetIgnoreDegradedGroupsChecks(true);
758+
request.SetIgnoreDisintegratedGroupsChecks(true);
759+
auto *cmd = request.AddCommand();
760+
auto *wipe = cmd->MutableWipeVDisk();
761+
auto *vslot = wipe->MutableVSlotId();
759762
vslot->SetNodeId(nodeId);
760763
vslot->SetPDiskId(pdiskId);
761764
vslot->SetVSlotId(vslotId);
762-
Runtime->SendToPipe(TabletId, self, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
763-
auto response = WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerGroupReconfigureWipeResult>(self);
764-
UNIT_ASSERT_VALUES_EQUAL(response->Get()->Record.GetStatus(), NKikimrProto::OK);
765+
VDiskIDFromVDiskID(vdiskId, wipe->MutableVDiskId());
766+
auto response = Invoke(request);
767+
UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription());
765768
}
766769

767770
void WaitForVDiskToGetRunning(const TVDiskID& vdiskId, TActorId actorId) {

‎ydb/core/blobstorage/ut_blobstorage/mirror3of4.cpp

+2-18
Original file line numberDiff line numberDiff line change
@@ -65,26 +65,10 @@ Y_UNIT_TEST_SUITE(Mirror3of4) {
6565
UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, NKikimrProto::OK);
6666
}
6767
if (i == 500) {
68-
const TActorId self = runtime->AllocateEdgeActor(1);
69-
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGroupReconfigureWipe>();
70-
ev->Record.MutableVSlotId()->SetNodeId(2);
71-
ev->Record.MutableVSlotId()->SetPDiskId(1000);
72-
ev->Record.MutableVSlotId()->SetVSlotId(1000);
73-
runtime->SendToPipe(env.TabletId, self, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
74-
auto response = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerGroupReconfigureWipeResult>(self);
75-
auto& r = response->Get()->Record;
76-
UNIT_ASSERT_EQUAL(r.GetStatus(), NKikimrProto::OK);
68+
env.Wipe(2, 1000, 1000, TVDiskID(groupId, 1, 0, 1, 0));
7769
}
7870
if (i == 600) {
79-
const TActorId self = runtime->AllocateEdgeActor(1);
80-
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGroupReconfigureWipe>();
81-
ev->Record.MutableVSlotId()->SetNodeId(3);
82-
ev->Record.MutableVSlotId()->SetPDiskId(1000);
83-
ev->Record.MutableVSlotId()->SetVSlotId(1000);
84-
runtime->SendToPipe(env.TabletId, self, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
85-
auto response = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerGroupReconfigureWipeResult>(self);
86-
auto& r = response->Get()->Record;
87-
UNIT_ASSERT_EQUAL(r.GetStatus(), NKikimrProto::OK);
71+
env.Wipe(3, 1000, 1000, TVDiskID(groupId, 1, 0, 2, 0));
8872
}
8973
}
9074

‎ydb/core/blobstorage/ut_blobstorage/osiris.cpp

+2-9
Original file line numberDiff line numberDiff line change
@@ -62,15 +62,8 @@ bool DoTestCase(TBlobStorageGroupType::EErasureSpecies erasure, const std::set<s
6262
for (const auto& vslot : response.GetStatus(0).GetBaseConfig().GetVSlot()) {
6363
const TVDiskID vdiskId(vslot.GetGroupId(), vslot.GetGroupGeneration(), vslot.GetFailRealmIdx(),
6464
vslot.GetFailDomainIdx(), vslot.GetVDiskIdx());
65-
66-
const TActorId sender = env.Runtime->AllocateEdgeActor(1);
67-
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGroupReconfigureWipe>();
68-
auto *slotId = ev->Record.MutableVSlotId();
69-
slotId->CopyFrom(vslot.GetVSlotId());
70-
env.Runtime->SendToPipe(env.TabletId, sender, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
71-
auto response = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerGroupReconfigureWipeResult>(sender);
72-
Y_ABORT_UNLESS(response->Get()->Record.GetStatus() == NKikimrProto::OK);
73-
65+
const auto& v = vslot.GetVSlotId();
66+
env.Wipe(v.GetNodeId(), v.GetPDiskId(), v.GetVSlotId(), vdiskId);
7467
env.Sim(TDuration::Seconds(30));
7568
}
7669

‎ydb/core/blobstorage/vdisk/common/blobstorage_event_filter.cpp

-6
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,6 @@ namespace NKikimr {
5454
TEvBlobStorage::EvControllerUpdateDiskStatus,
5555
TEvBlobStorage::EvControllerConfigRequest,
5656
TEvBlobStorage::EvControllerConfigResponse,
57-
TEvBlobStorage::EvControllerVDiskStatusSubscribeRequest,
58-
TEvBlobStorage::EvControllerVDiskStatusReport,
59-
TEvBlobStorage::EvControllerGroupStatusRequest,
60-
TEvBlobStorage::EvControllerGroupStatusResponse,
61-
TEvBlobStorage::EvControllerUpdateGroup,
62-
TEvBlobStorage::EvControllerUpdateFaultyDisks,
6357
TEvBlobStorage::EvControllerUpdateGroupStat,
6458

6559
TEvBlobStorage::EvControllerSelectGroupsResult,

‎ydb/core/mind/bscontroller/config.cpp

+12-10
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,21 @@ namespace NKikimr::NBsController {
1717
, State(state)
1818
{}
1919

20-
void Execute(std::deque<std::unique_ptr<IEventHandle>>& outbox) {
20+
void Execute() {
2121
ApplyUpdates();
2222

2323
for (auto &pair : Services) {
2424
const TNodeId &nodeId = pair.first;
2525

26-
if (TNodeInfo *node = Self->FindNode(nodeId); node && node->ConnectedCount) {
27-
auto event = MakeHolder<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>();
26+
if (TNodeInfo *node = Self->FindNode(nodeId); node && node->ConnectedServerId) {
27+
auto event = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>();
2828
auto& record = event->Record;
2929
pair.second.Swap(&record);
3030
record.SetStatus(NKikimrProto::OK);
3131
record.SetNodeID(nodeId);
3232
record.SetInstanceId(Self->InstanceId);
3333
record.SetAvailDomain(AppData()->DomainsInfo->GetDomainUidByTabletId(Self->TabletID()));
34-
outbox.push_back(std::make_unique<IEventHandle>(MakeBlobStorageNodeWardenID(nodeId),
35-
Self->SelfId(), event.Release()));
34+
State.Outbox.emplace_back(nodeId, std::move(event), 0);
3635
}
3736
}
3837
}
@@ -448,7 +447,7 @@ namespace NKikimr::NBsController {
448447
}
449448
}
450449

451-
TNodeWardenUpdateNotifier(this, state).Execute(state.Outbox);
450+
TNodeWardenUpdateNotifier(this, state).Execute();
452451

453452
state.CheckConsistency();
454453
state.Commit();
@@ -460,7 +459,7 @@ namespace NKikimr::NBsController {
460459
}
461460

462461
void TBlobStorageController::CommitSelfHealUpdates(TConfigState& state) {
463-
auto ev = MakeHolder<TEvControllerNotifyGroupChange>();
462+
auto ev = std::make_unique<TEvControllerNotifyGroupChange>();
464463
auto sh = MakeHolder<TEvControllerUpdateSelfHealInfo>();
465464

466465
for (auto&& [base, overlay] : state.Groups.Diff()) {
@@ -500,7 +499,7 @@ namespace NKikimr::NBsController {
500499
}
501500

502501
if (ev->Created || ev->Deleted) {
503-
state.Outbox.push_back(std::make_unique<IEventHandle>(StatProcessorActorId, SelfId(), ev.Release()));
502+
state.StatProcessorOutbox.push_back(std::move(ev));
504503
}
505504
if (sh->GroupsToUpdate) {
506505
FillInSelfHealGroups(*sh, &state);
@@ -590,8 +589,11 @@ namespace NKikimr::NBsController {
590589
}
591590

592591
ui64 TBlobStorageController::TConfigState::ApplyConfigUpdates() {
593-
for (auto& msg : Outbox) {
594-
TActivationContext::Send(msg.release());
592+
for (auto& [nodeId, ev, cookie] : Outbox) {
593+
Self.SendToWarden(nodeId, std::move(ev), cookie);
594+
}
595+
for (auto& ev : StatProcessorOutbox) {
596+
Self.SelfId().Send(Self.StatProcessorActorId, ev.release());
595597
}
596598

597599
if (UpdateSelfHealInfoMsg) {

‎ydb/core/mind/bscontroller/config.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,8 @@ namespace NKikimr {
8989
THashSet<TPDiskId> PDisksToRemove;
9090

9191
// outgoing messages
92-
std::deque<std::unique_ptr<IEventHandle>> Outbox;
92+
std::deque<std::tuple<TNodeId, std::unique_ptr<IEventBase>, ui64>> Outbox;
93+
std::deque<std::unique_ptr<IEventBase>> StatProcessorOutbox;
9394
THolder<TEvControllerUpdateSelfHealInfo> UpdateSelfHealInfoMsg;
9495

9596
// deferred callbacks

‎ydb/core/mind/bscontroller/get_group.cpp

+18-10
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ namespace NKikimr::NBsController {
44

55
class TBlobStorageController::TTxGetGroup : public TTransactionBase<TBlobStorageController> {
66
TEvBlobStorage::TEvControllerGetGroup::TPtr Request;
7-
std::unique_ptr<IEventHandle> Response;
7+
std::unique_ptr<TEvBlobStorage::TEvControllerNodeServiceSetUpdate> Response;
8+
TNodeId NodeId = {};
89

910
public:
1011
TTxGetGroup(TEvBlobStorage::TEvControllerGetGroup::TPtr& ev, TBlobStorageController *controller)
@@ -20,27 +21,34 @@ class TBlobStorageController::TTxGetGroup : public TTransactionBase<TBlobStorage
2021

2122
STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXGG01, "Handle TEvControllerGetGroup", (Request, Request->Get()->Record));
2223

24+
NodeId = Request->Get()->Record.GetNodeID();
25+
26+
if (Request->Cookie != Max<ui64>() && !Self->ValidateIncomingNodeWardenEvent(*Request)) {
27+
Response = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>(NKikimrProto::ERROR, NodeId);
28+
return true;
29+
}
30+
2331
const auto& v = Request->Get()->Record.GetGroupIDs();
2432
TSet<ui32> groupIDsToRead(v.begin(), v.end());
2533

26-
const TNodeId nodeId = Request->Get()->Record.GetNodeID();
27-
auto res = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>(NKikimrProto::OK, nodeId);
28-
Self->ReadGroups(groupIDsToRead, true, res.get(), nodeId);
34+
Response = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>(NKikimrProto::OK, NodeId);
35+
Self->ReadGroups(groupIDsToRead, true, Response.get(), NodeId);
2936

30-
auto& node = Self->GetNode(nodeId);
37+
auto& node = Self->GetNode(NodeId);
3138
for (TGroupId groupId : v) {
3239
node.GroupsRequested.insert(groupId);
33-
Self->GroupToNode.emplace(groupId, nodeId);
40+
Self->GroupToNode.emplace(groupId, NodeId);
3441
}
3542

36-
Response = std::make_unique<IEventHandle>(nodeId ? MakeBlobStorageNodeWardenID(nodeId) : Request->Sender,
37-
Self->SelfId(), res.release());
38-
3943
return true;
4044
}
4145

4246
void Complete(const TActorContext&) override {
43-
TActivationContext::Send(Response.release());
47+
if (NodeId) {
48+
Self->SendToWarden(NodeId, std::move(Response), Request->Cookie);
49+
} else {
50+
Self->SendInReply(*Request, std::move(Response));
51+
}
4452
}
4553
};
4654

0 commit comments

Comments
 (0)