From f59e0231a2aa6492a81b60154019207b6d4ba13c Mon Sep 17 00:00:00 2001 From: Semyon Danilov Date: Thu, 16 Jan 2025 22:02:34 +0400 Subject: [PATCH 1/2] Add node interlace to GroupMapper's TScode --- .../mind/bscontroller/group_layout_checker.h | 18 ++++++++-- ydb/core/mind/bscontroller/group_mapper.cpp | 4 +-- .../mind/bscontroller/group_mapper_ut.cpp | 33 +++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/ydb/core/mind/bscontroller/group_layout_checker.h b/ydb/core/mind/bscontroller/group_layout_checker.h index 78c9e3e05f86..b2345f40ec8c 100644 --- a/ydb/core/mind/bscontroller/group_layout_checker.h +++ b/ydb/core/mind/bscontroller/group_layout_checker.h @@ -72,13 +72,15 @@ namespace NKikimr::NBsController { TEntityId RealmGroup; TEntityId Realm; TEntityId Domain; + TEntityId Node; TPDiskLayoutPosition() = default; - TPDiskLayoutPosition(TEntityId realmGroup, TEntityId realm, TEntityId domain) + TPDiskLayoutPosition(TEntityId realmGroup, TEntityId realm, TEntityId domain, TEntityId node) : RealmGroup(realmGroup) , Realm(realm) , Domain(domain) + , Node(node) {} TPDiskLayoutPosition(TDomainMapper& mapper, const TNodeLocation& location, TPDiskId pdiskId, const TGroupGeometryInfo& geom) { @@ -102,6 +104,7 @@ namespace NKikimr::NBsController { RealmGroup = mapper(realmGroup.Str()); Realm = mapper(realm.Str()); Domain = mapper(domain.Str()); + Node.Value = pdiskId.NodeId; } TString ToString() const { @@ -124,12 +127,13 @@ namespace NKikimr::NBsController { struct TScore { ui32 RealmInterlace = 0; ui32 DomainInterlace = 0; + ui32 NodeInterlace = 0; ui32 RealmGroupScatter = 0; ui32 RealmScatter = 0; ui32 DomainScatter = 0; auto AsTuple() const { - return std::make_tuple(RealmInterlace, DomainInterlace, RealmGroupScatter, RealmScatter, DomainScatter); + return std::make_tuple(RealmInterlace, DomainInterlace, NodeInterlace, RealmGroupScatter, RealmScatter, DomainScatter); } bool BetterThan(const TScore& other) const { @@ -141,12 +145,13 @@ namespace NKikimr::NBsController { } static TScore Max() { - return {::Max(), ::Max(), ::Max(), ::Max(), ::Max()}; + return {::Max(), ::Max(), ::Max(), ::Max(), ::Max(), ::Max()}; } TString ToString() const { return TStringBuilder() << "{RealmInterlace# " << RealmInterlace << " DomainInterlace# " << DomainInterlace + << " NodeInterlace# " << NodeInterlace << " RealmGroupScatter# " << RealmGroupScatter << " RealmScatter# " << RealmScatter << " DomainScatter# " << DomainScatter @@ -168,6 +173,8 @@ namespace NKikimr::NBsController { TStackVec, 32> NumDisksPerDomain; THashMap NumDisksPerDomainTotal; + THashMap NumDisksPerNode; + TGroupLayout(const TBlobStorageGroupInfo::TTopology& topology) : Topology(topology) , NumDisksInRealm(Topology.GetTotalFailRealmsNum()) @@ -187,6 +194,8 @@ namespace NKikimr::NBsController { NumDisksInDomain[domainIdx] += value; NumDisksPerDomain[domainIdx][pos.Domain] += value; NumDisksPerDomainTotal[pos.Domain] += value; + + NumDisksPerNode[pos.Node] += value; } void AddDisk(const TPDiskLayoutPosition& pos, ui32 orderNumber) { @@ -204,9 +213,12 @@ namespace NKikimr::NBsController { const auto& disksPerRealm = NumDisksPerRealm[vdisk.FailRealm][pos.Realm]; const auto& disksPerDomain = NumDisksPerDomain[domainIdx][pos.Domain]; + const ui32 disksOnNode = NumDisksPerNode[pos.Node]; + return { .RealmInterlace = NumDisksPerRealmTotal[pos.Realm] - disksPerRealm, .DomainInterlace = NumDisksPerDomainTotal[pos.Domain] - disksPerDomain, + .NodeInterlace = disksOnNode, .RealmGroupScatter = NumDisks - NumDisksPerRealmGroup[pos.RealmGroup], .RealmScatter = NumDisksInRealm[vdisk.FailRealm] - disksPerRealm, .DomainScatter = NumDisksInDomain[domainIdx] - disksPerDomain, diff --git a/ydb/core/mind/bscontroller/group_mapper.cpp b/ydb/core/mind/bscontroller/group_mapper.cpp index c31fd5c83c51..37942fd36e47 100644 --- a/ydb/core/mind/bscontroller/group_mapper.cpp +++ b/ydb/core/mind/bscontroller/group_mapper.cpp @@ -405,7 +405,7 @@ namespace NKikimr::NBsController { static std::pair MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) { scope = x.Realm; - return {{x.RealmGroup, x.Realm, TEntityId::Min()}, {x.RealmGroup, x.Realm, TEntityId::Max()}}; + return {{x.RealmGroup, x.Realm, TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, x.Realm, TEntityId::Max(), TEntityId::Max()}}; } }; @@ -415,7 +415,7 @@ namespace NKikimr::NBsController { static std::pair MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) { scope = x.RealmGroup; - return {{x.RealmGroup, TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, TEntityId::Max(), TEntityId::Max()}}; + return {{x.RealmGroup, TEntityId::Min(), TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, TEntityId::Max(), TEntityId::Max(), TEntityId::Max()}}; } }; diff --git a/ydb/core/mind/bscontroller/group_mapper_ut.cpp b/ydb/core/mind/bscontroller/group_mapper_ut.cpp index 8126b2d53101..9b910bbcd1a1 100644 --- a/ydb/core/mind/bscontroller/group_mapper_ut.cpp +++ b/ydb/core/mind/bscontroller/group_mapper_ut.cpp @@ -638,6 +638,39 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) { } } + Y_UNIT_TEST(InterlacedRacksWithoutInterlacedNodes) { + TTestContext context( + { + {1, 1, 1, 1, 1}, // node 1 + {1, 1, 2, 1, 1}, + {1, 1, 3, 1, 2}, // node 3 has two disks + {1, 1, 4, 1, 1}, + {1, 1, 5, 1, 1}, + {1, 1, 6, 1, 1}, + {1, 1, 2, 1, 1}, // node 7 is in the same rack as node 2 + {1, 1, 8, 1, 1}, + {1, 1, 3, 1, 1}, // node 9 is in the same rack as node 3 + } + ); + + TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::Erasure4Plus2Block)); + context.PopulateGroupMapper(mapper, 8); + + TGroupMapper::TGroupDefinition group; + group.emplace_back(TVector>(8)); + auto& g = group[0]; + + for (int i = 0; i < 8; i++) { + g[i].emplace_back(TPDiskId(i + 1, 1)); + } + + context.SetGroup(1, group); + + TGroupMapper::TGroupDefinition newGroup = context.ReallocateGroup(mapper, 1, {TPDiskId(8, 1)}); + + UNIT_ASSERT_EQUAL_C(TPDiskId(9, 1), newGroup[0][7][0], context.FormatGroup(newGroup)); + } + Y_UNIT_TEST(NonUniformClusterDifferentSlotsPerDisk) { std::vector> disks; for (ui32 rack = 0; rack < 12; ++rack) { From f28b503fa32e5bbf1c7fbdf4d83f3b6ca56b612a Mon Sep 17 00:00:00 2001 From: Semyon Danilov Date: Mon, 20 Jan 2025 19:44:26 +0400 Subject: [PATCH 2/2] Remove odd test --- ydb/core/mind/bscontroller/group_mapper_ut.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ydb/core/mind/bscontroller/group_mapper_ut.cpp b/ydb/core/mind/bscontroller/group_mapper_ut.cpp index 9b910bbcd1a1..35756f9fc0c8 100644 --- a/ydb/core/mind/bscontroller/group_mapper_ut.cpp +++ b/ydb/core/mind/bscontroller/group_mapper_ut.cpp @@ -593,10 +593,6 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) { TestBlock42(1); } - Y_UNIT_TEST(Block42_2disk) { - TestBlock42(2); - } - Y_UNIT_TEST(Mirror3dc) { TTestContext context(6, 3, 3, 3, 3); TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::ErasureMirror3dc));