Reset replication counters correctly to avoid data race on VDisk restart (#5486)

Author: alexvru

ydb/core/blobstorage/nodewarden/node_warden_impl.h

@@ -335,6 +335,9 @@ namespace NKikimr::NStorage {
                 bool ReadOnly;
             };
             std::optional<TRuntimeData> RuntimeData;
+            bool ShutdownPending = false;
+            bool RestartAfterShutdown = false;
+            TDuration YardInitDelay;
 
             // Last VDiskId reported to Node Whiteboard.
             std::optional<TVDiskID> WhiteboardVDiskId;
@@ -386,13 +389,15 @@ namespace NKikimr::NStorage {
         };
 
         std::map<TVSlotId, TVDiskRecord> LocalVDisks;
+        THashMap<TActorId, TVSlotId> VDiskIdByActor;
         std::map<TVSlotId, ui64> SlayInFlight;
         std::set<ui32> PDiskRestartInFlight;
         TIntrusiveList<TVDiskRecord, TUnreportedMetricTag> VDisksWithUnreportedMetrics;
 
         void DestroyLocalVDisk(TVDiskRecord& vdisk);
         void PoisonLocalVDisk(TVDiskRecord& vdisk);
         void StartLocalVDiskActor(TVDiskRecord& vdisk, TDuration yardInitDelay);
+        void HandleGone(STATEFN_SIG);
         void ApplyServiceSetVDisks(const NKikimrBlobStorage::TNodeWardenServiceSet& serviceSet);
 
         // process VDisk configuration
@@ -646,6 +651,8 @@ namespace NKikimr::NStorage {
                 hFunc(TEvNodeWardenQueryBaseConfig, Handle);
                 hFunc(TEvNodeConfigInvokeOnRootResult, Handle);
 
+                fFunc(TEvents::TSystem::Gone, HandleGone);
+
                 default:
                     EnqueuePendingMessage(ev);
                     break;

ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp

@@ -48,6 +48,7 @@ namespace NKikimr::NStorage {
         vdisk.ScrubCookie = 0; // disable reception of Scrub messages from this disk
         vdisk.ScrubCookieForController = 0; // and from controller too
         vdisk.Status = NKikimrBlobStorage::EVDiskStatus::ERROR;
+        vdisk.ShutdownPending = true;
 
         SendDiskMetrics(false);
     }
@@ -70,6 +71,12 @@ namespace NKikimr::NStorage {
             return;
         }
 
+        if (vdisk.ShutdownPending) {
+            vdisk.RestartAfterShutdown = true;
+            vdisk.YardInitDelay = Max(vdisk.YardInitDelay, yardInitDelay);
+            return;
+        }
+
         // find underlying PDisk and determine its media type
         auto pdiskIt = LocalPDisks.find({vslotId.NodeId, vslotId.PDiskId});
         Y_VERIFY_S(pdiskIt != LocalPDisks.end(), "PDiskId# " << vslotId.NodeId << ":" << vslotId.PDiskId << " not found");
@@ -206,8 +213,10 @@ namespace NKikimr::NStorage {
 
         // create an actor
         auto *as = TActivationContext::ActorSystem();
-        as->RegisterLocalService(vdiskServiceId, as->Register(CreateVDisk(vdiskConfig, groupInfo, AppData()->Counters),
-            TMailboxType::Revolving, AppData()->SystemPoolId));
+        TActorId actorId = as->Register(CreateVDisk(vdiskConfig, groupInfo, AppData()->Counters),
+            TMailboxType::Revolving, AppData()->SystemPoolId);
+        as->RegisterLocalService(vdiskServiceId, actorId);
+        VDiskIdByActor.try_emplace(actorId, vslotId);
 
         STLOG(PRI_DEBUG, BS_NODE, NW24, "StartLocalVDiskActor done", (VDiskId, vdisk.GetVDiskId()), (VSlotId, vslotId),
             (PDiskGuid, pdiskGuid));
@@ -233,6 +242,22 @@ namespace NKikimr::NStorage {
         vdisk.ScrubCookie = scrubCookie;
     }
 
+    void TNodeWarden::HandleGone(STATEFN_SIG) {
+        if (const auto it = VDiskIdByActor.find(ev->Sender); it != VDiskIdByActor.end()) {
+            if (const auto jt = LocalVDisks.find(it->second); jt != LocalVDisks.end()) {
+                TVDiskRecord& vdisk = jt->second;
+                Y_ABORT_UNLESS(vdisk.ShutdownPending);
+                vdisk.ShutdownPending = false;
+                if (vdisk.RestartAfterShutdown) {
+                    StartLocalVDiskActor(vdisk, vdisk.YardInitDelay);
+                    vdisk.RestartAfterShutdown = false;
+                    vdisk.YardInitDelay = TDuration::Zero();
+                }
+            }
+            VDiskIdByActor.erase(it);
+        }
+    }
+
     void TNodeWarden::ApplyServiceSetVDisks(const NKikimrBlobStorage::TNodeWardenServiceSet& serviceSet) {
         for (const auto& vdisk : serviceSet.GetVDisks()) {
             ApplyLocalVDiskInfo(vdisk);

ydb/core/blobstorage/ut_group/main.cpp

@@ -123,6 +123,7 @@ class TNodeWardenMockActor : public TActor<TNodeWardenMockActor> {
         cFunc(TEvBlobStorage::EvDropDonor, Ignore);
         cFunc(TEvBlobStorage::EvGroupStatReport, Ignore);
         cFunc(TEvBlobStorage::EvNotifyVDiskGenerationChange, Ignore);
+        cFunc(TEvents::TSystem::Gone, Ignore);
 
         fFunc(TEvBlobStorage::EvPut, ForwardToProxy);
         fFunc(TEvBlobStorage::EvGet, ForwardToProxy);

ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp

@@ -2209,6 +2209,8 @@ namespace NKikimr {
 
         void PassAway() override {
             VDiskCountersBase->RemoveSubgroupChain(CountersChain);
+            TActivationContext::Send(new IEventHandle(TEvents::TSystem::Gone, 0,
+                MakeBlobStorageNodeWardenID(SelfId().NodeId()), SelfId(), nullptr, 0));
             TActorBootstrapped::PassAway();
         }
     };