Add node interlace to GroupMapper's TScore (#13458)

Author: SammyVimes

ydb/core/mind/bscontroller/group_layout_checker.h

@@ -72,13 +72,15 @@ namespace NKikimr::NBsController {
             TEntityId RealmGroup;
             TEntityId Realm;
             TEntityId Domain;
+            TEntityId Node;
 
             TPDiskLayoutPosition() = default;
 
-            TPDiskLayoutPosition(TEntityId realmGroup, TEntityId realm, TEntityId domain)
+            TPDiskLayoutPosition(TEntityId realmGroup, TEntityId realm, TEntityId domain, TEntityId node)
                 : RealmGroup(realmGroup)
                 , Realm(realm)
                 , Domain(domain)
+                , Node(node)
             {}
 
             TPDiskLayoutPosition(TDomainMapper& mapper, const TNodeLocation& location, TPDiskId pdiskId, const TGroupGeometryInfo& geom) {
@@ -102,6 +104,7 @@ namespace NKikimr::NBsController {
                 RealmGroup = mapper(realmGroup.Str());
                 Realm = mapper(realm.Str());
                 Domain = mapper(domain.Str());
+                Node.Value = pdiskId.NodeId;
             }
 
             TString ToString() const {
@@ -124,12 +127,13 @@ namespace NKikimr::NBsController {
         struct TScore {
             ui32 RealmInterlace = 0;
             ui32 DomainInterlace = 0;
+            ui32 NodeInterlace = 0;
             ui32 RealmGroupScatter = 0;
             ui32 RealmScatter = 0;
             ui32 DomainScatter = 0;
 
             auto AsTuple() const {
-                return std::make_tuple(RealmInterlace, DomainInterlace, RealmGroupScatter, RealmScatter, DomainScatter);
+                return std::make_tuple(RealmInterlace, DomainInterlace, NodeInterlace, RealmGroupScatter, RealmScatter, DomainScatter);
             }
 
             bool BetterThan(const TScore& other) const {
@@ -141,12 +145,13 @@ namespace NKikimr::NBsController {
             }
 
             static TScore Max() {
-                return {::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>()};
+                return {::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>(), ::Max<ui32>()};
             }
 
             TString ToString() const {
                 return TStringBuilder() << "{RealmInterlace# " << RealmInterlace
                     << " DomainInterlace# " << DomainInterlace
+                    << " NodeInterlace# " << NodeInterlace
                     << " RealmGroupScatter# " << RealmGroupScatter
                     << " RealmScatter# " << RealmScatter
                     << " DomainScatter# " << DomainScatter
@@ -168,6 +173,8 @@ namespace NKikimr::NBsController {
             TStackVec<THashMap<TEntityId, ui32>, 32> NumDisksPerDomain;
             THashMap<TEntityId, ui32> NumDisksPerDomainTotal;
 
+            THashMap<TEntityId, ui32> NumDisksPerNode;
+
             TGroupLayout(const TBlobStorageGroupInfo::TTopology& topology)
                 : Topology(topology)
                 , NumDisksInRealm(Topology.GetTotalFailRealmsNum())
@@ -187,6 +194,8 @@ namespace NKikimr::NBsController {
                 NumDisksInDomain[domainIdx] += value;
                 NumDisksPerDomain[domainIdx][pos.Domain] += value;
                 NumDisksPerDomainTotal[pos.Domain] += value;
+
+                NumDisksPerNode[pos.Node] += value;
             }
 
             void AddDisk(const TPDiskLayoutPosition& pos, ui32 orderNumber) {
@@ -204,9 +213,12 @@ namespace NKikimr::NBsController {
                 const auto& disksPerRealm = NumDisksPerRealm[vdisk.FailRealm][pos.Realm];
                 const auto& disksPerDomain = NumDisksPerDomain[domainIdx][pos.Domain];
 
+                const ui32 disksOnNode = NumDisksPerNode[pos.Node];
+
                 return {
                     .RealmInterlace = NumDisksPerRealmTotal[pos.Realm] - disksPerRealm,
                     .DomainInterlace = NumDisksPerDomainTotal[pos.Domain] - disksPerDomain,
+                    .NodeInterlace = disksOnNode,
                     .RealmGroupScatter = NumDisks - NumDisksPerRealmGroup[pos.RealmGroup],
                     .RealmScatter = NumDisksInRealm[vdisk.FailRealm] - disksPerRealm,
                     .DomainScatter = NumDisksInDomain[domainIdx] - disksPerDomain,

ydb/core/mind/bscontroller/group_mapper.cpp

@@ -405,7 +405,7 @@ namespace NKikimr::NBsController {
 
                 static std::pair<TPDiskLayoutPosition, TPDiskLayoutPosition> MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) {
                     scope = x.Realm;
-                    return {{x.RealmGroup, x.Realm, TEntityId::Min()}, {x.RealmGroup, x.Realm, TEntityId::Max()}};
+                    return {{x.RealmGroup, x.Realm, TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, x.Realm, TEntityId::Max(), TEntityId::Max()}};
                 }
             };
 
@@ -415,7 +415,7 @@ namespace NKikimr::NBsController {
 
                 static std::pair<TPDiskLayoutPosition, TPDiskLayoutPosition> MakeRange(const TPDiskLayoutPosition& x, TEntityId& scope) {
                     scope = x.RealmGroup;
-                    return {{x.RealmGroup, TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, TEntityId::Max(), TEntityId::Max()}};
+                    return {{x.RealmGroup, TEntityId::Min(), TEntityId::Min(), TEntityId::Min()}, {x.RealmGroup, TEntityId::Max(), TEntityId::Max(), TEntityId::Max()}};
                 }
             };
 

ydb/core/mind/bscontroller/group_mapper_ut.cpp

@@ -593,10 +593,6 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) {
         TestBlock42(1);
     }
 
-    Y_UNIT_TEST(Block42_2disk) {
-        TestBlock42(2);
-    }
-
     Y_UNIT_TEST(Mirror3dc) {
         TTestContext context(6, 3, 3, 3, 3);
         TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::ErasureMirror3dc));
@@ -638,6 +634,39 @@ Y_UNIT_TEST_SUITE(TGroupMapperTest) {
         }
     }
 
+    Y_UNIT_TEST(InterlacedRacksWithoutInterlacedNodes) {
+        TTestContext context(
+            {
+                {1, 1, 1, 1, 1}, // node 1
+                {1, 1, 2, 1, 1},
+                {1, 1, 3, 1, 2}, // node 3 has two disks
+                {1, 1, 4, 1, 1},
+                {1, 1, 5, 1, 1},
+                {1, 1, 6, 1, 1},
+                {1, 1, 2, 1, 1}, // node 7 is in the same rack as node 2
+                {1, 1, 8, 1, 1},
+                {1, 1, 3, 1, 1}, // node 9 is in the same rack as node 3 
+            }
+        );
+
+        TGroupMapper mapper(TTestContext::CreateGroupGeometry(TBlobStorageGroupType::Erasure4Plus2Block));
+        context.PopulateGroupMapper(mapper, 8);
+        
+        TGroupMapper::TGroupDefinition group;
+        group.emplace_back(TVector<TVector<TPDiskId>>(8));
+        auto& g = group[0];
+
+        for (int i = 0; i < 8; i++) {
+            g[i].emplace_back(TPDiskId(i + 1, 1));
+        }
+
+        context.SetGroup(1, group);
+
+        TGroupMapper::TGroupDefinition newGroup = context.ReallocateGroup(mapper, 1, {TPDiskId(8, 1)});
+
+        UNIT_ASSERT_EQUAL_C(TPDiskId(9, 1), newGroup[0][7][0], context.FormatGroup(newGroup));
+    }
+
     Y_UNIT_TEST(NonUniformClusterDifferentSlotsPerDisk) {
         std::vector<std::tuple<ui32, ui32, ui32, ui32, ui32>> disks;
         for (ui32 rack = 0; rack < 12; ++rack) {