Correct temporary PDisk start sequence (#7156)

Author: alexvru

ydb/core/blobstorage/nodewarden/distconf_persistent_storage.cpp

@@ -37,13 +37,22 @@ namespace NKikimr::NStorage {
                 const size_t index = ev->Cookie;
                 Y_ABORT_UNLESS(index < Paths.size());
                 const TString& path = Paths[index];
+                auto& record = msg->Record;
 
                 STLOG(PRI_DEBUG, BS_NODE, NWDC50, "TReaderActor result", (Path, path), (Outcome, msg->Outcome),
-                    (Guid, msg->Guid), (Record, msg->Record));
+                    (Guid, msg->Guid), (Record, record));
 
                 switch (msg->Outcome) {
                     case NPDisk::EPDiskMetadataOutcome::OK:
-                        Response->MetadataPerPath.emplace_back(path, std::move(msg->Record), msg->Guid);
+                        if (record.HasProposedStorageConfig() && !CheckFingerprint(record.GetProposedStorageConfig())) {
+                            Y_DEBUG_ABORT("ProposedStorageConfig metadata fingerprint incorrect");
+                            Response->Errors.push_back(path);
+                        } else if (record.HasCommittedStorageConfig() && !CheckFingerprint(record.GetCommittedStorageConfig())) {
+                            Y_DEBUG_ABORT("CommittedStorageConfig metadata fingerprint incorrect");
+                            Response->Errors.push_back(path);
+                        } else {
+                            Response->MetadataPerPath.emplace_back(path, std::move(record), msg->Guid);
+                        }
                         break;
 
                     case NPDisk::EPDiskMetadataOutcome::NO_METADATA:

ydb/core/blobstorage/nodewarden/node_warden_impl.h

@@ -108,6 +108,7 @@ namespace NKikimr::NStorage {
             std::optional<NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk> Pending; // pending
         };
         THashMap<TString, TPDiskByPathInfo> PDiskByPath;
+        THashSet<ui32> PDisksWaitingToStart;
 
         ui64 LastScrubCookie = RandomNumber<ui64>();
 

ydb/core/blobstorage/nodewarden/node_warden_mon.cpp

@@ -134,25 +134,50 @@ void TNodeWarden::RenderWholePage(IOutputStream& out) {
                     TABLEH() { out << "Path"; }
                     TABLEH() { out << "Guid"; }
                     TABLEH() { out << "Category"; }
+                    TABLEH() { out << "Temporary"; }
+                    TABLEH() { out << "Pending"; }
                 }
             }
             TABLEBODY() {
                 for (auto& [key, value] : LocalPDisks) {
+                    TString pending;
+                    if (const auto it = PDiskByPath.find(value.Record.GetPath()); it != PDiskByPath.end()) {
+                        if (it->second.Pending) {
+                            pending = TStringBuilder() << "PDiskId# " << it->second.Pending->GetPDiskID();
+                        } else {
+                            pending = "<none>";
+                        }
+                    } else {
+                        pending = "<path not found>"; // this is strange
+                    }
+
                     TABLER() {
-                        TABLED() { out << "(" << key.NodeId << "," << key.PDiskId << ")"; }
+                        TABLED() { out << "[" << key.NodeId << ":" << key.PDiskId << "]"; }
                         TABLED() { out << value.Record.GetPath(); }
                         TABLED() { out << value.Record.GetPDiskGuid(); }
                         TABLED() { out << value.Record.GetPDiskCategory(); }
+                        TABLED() { out << value.Temporary; }
+                        TABLED() { out << pending; }
                     }
                 }
             }
         }
+        if (!PDiskRestartInFlight.empty()) {
+            DIV() {
+                out << "PDiskRestartInFlight# " << FormatList(PDiskRestartInFlight);
+            }
+        }
+        if (!PDisksWaitingToStart.empty()) {
+            DIV() {
+                out << "PDisksWaitingToStart# " << FormatList(PDisksWaitingToStart);
+            }
+        }
 
         TAG(TH3) { out << "VDisks"; }
         TABLE_CLASS("table oddgray") {
             TABLEHEAD() {
                 TABLER() {
-                    TABLEH() { out << "Location (NodeId, PDiskId, VSlotId)"; }
+                    TABLEH() { out << "Location"; }
                     TABLEH() { out << "VDiskId"; }
                     TABLEH() { out << "Running"; }
                     TABLEH() { out << "StoragePoolName"; }
@@ -164,7 +189,7 @@ void TNodeWarden::RenderWholePage(IOutputStream& out) {
             TABLEBODY() {
                 for (auto& [key, value] : LocalVDisks) {
                     TABLER() {
-                        TABLED() { out << "(" << key.NodeId << "," << key.PDiskId << "," << key.VDiskSlotId << ")"; }
+                        TABLED() { out << "[" << key.NodeId << ":" << key.PDiskId << ":" << key.VDiskSlotId << "]"; }
                         TABLED() { out << value.GetVDiskId(); }
                         TABLED() { out << (value.RuntimeData ? "true" : "false"); }
                         TABLED() { out << value.Config.GetStoragePoolName(); }

ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp

@@ -134,6 +134,8 @@ namespace NKikimr::NStorage {
             Y_ABORT_UNLESS(jt != LocalPDisks.end());
             TPDiskRecord& record = jt->second;
             if (record.Temporary) { // this is temporary PDisk spinning, have to wait for it to finish
+                Y_ABORT_UNLESS(!temporary);
+                PDisksWaitingToStart.insert(pdisk.GetPDiskID());
                 it->second.Pending = pdisk;
             } else { // incorrect configuration: we are trying to start two different PDisks with the same path
                 STLOG(PRI_ERROR, BS_NODE, NW48, "starting two PDisks with the same path", (Path, path),
@@ -215,6 +217,7 @@ namespace NKikimr::NStorage {
                     jt->second.RunningPDiskId == it->first) {
                 pending = std::move(jt->second.Pending);
                 PDiskByPath.erase(jt);
+                PDisksWaitingToStart.erase(pending->GetPDiskID());
             } else {
                 Y_DEBUG_ABORT("missing entry in PDiskByPath");
             }
@@ -230,6 +233,16 @@ namespace NKikimr::NStorage {
 
         if (pending) {
             StartLocalPDisk(*pending, false);
+
+            // start VDisks over this one waiting for their turn
+            const ui32 actualPDiskId = pending->GetPDiskID();
+            const TVSlotId from(LocalNodeId, actualPDiskId, 0);
+            const TVSlotId to(LocalNodeId, actualPDiskId, Max<ui32>());
+            for (auto it = LocalVDisks.lower_bound(from); it != LocalVDisks.end() && it->first <= to; ++it) {
+                auto& [key, value] = *it;
+                Y_ABORT_UNLESS(!value.RuntimeData); // they can't be working
+                StartLocalVDiskActor(value);
+            }
         }
     }
 
@@ -300,7 +313,7 @@ namespace NKikimr::NStorage {
         } else {
             for (auto it = LocalVDisks.lower_bound(from); it != LocalVDisks.end() && it->first <= to; ++it) {
                 auto& [key, value] = *it;
-                if (!value.RuntimeData && !SlayInFlight.contains(key)) {
+                if (!value.RuntimeData) {
                     StartLocalVDiskActor(value);
                 }
             }
@@ -442,6 +455,7 @@ namespace NKikimr::NStorage {
         }
         for (const TString& path : pathsToResetPending) {
             if (const auto it = PDiskByPath.find(path); it != PDiskByPath.end()) {
+                PDisksWaitingToStart.erase(it->second.Pending->GetPDiskID());
                 it->second.Pending.reset();
             }
         }

ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp

@@ -60,7 +60,9 @@ namespace NKikimr::NStorage {
         Y_VERIFY_S(!donorMode || !readOnly, "Only one of modes should be enabled: donorMode " << donorMode << ", readOnly " << readOnly);
 
         STLOG(PRI_DEBUG, BS_NODE, NW23, "StartLocalVDiskActor", (SlayInFlight, SlayInFlight.contains(vslotId)),
-            (VDiskId, vdisk.GetVDiskId()), (VSlotId, vslotId), (PDiskGuid, pdiskGuid), (DonorMode, donorMode));
+            (VDiskId, vdisk.GetVDiskId()), (VSlotId, vslotId), (PDiskGuid, pdiskGuid), (DonorMode, donorMode),
+            (PDiskRestartInFlight, PDiskRestartInFlight.contains(vslotId.PDiskId)),
+            (PDisksWaitingToStart, PDisksWaitingToStart.contains(vslotId.PDiskId)));
 
         if (SlayInFlight.contains(vslotId)) {
             return;
@@ -70,6 +72,10 @@ namespace NKikimr::NStorage {
             return;
         }
 
+        if (PDisksWaitingToStart.contains(vslotId.PDiskId)) {
+            return;
+        }
+
         if (vdisk.ShutdownPending) {
             vdisk.RestartAfterShutdown = true;
             return;