Support metadata storage over pdisk (#6814)

Author: alexvru

ydb/core/base/blobstorage.h

@@ -732,6 +732,7 @@ struct TEvBlobStorage {
         EvMinHugeBlobSizeUpdate,
         EvHugePreCompact,
         EvHugePreCompactResult,
+        EvPDiskMetadataLoaded,
 
         EvYardInitResult = EvPut + 9 * 512,                     /// 268 636 672
         EvLogResult,
@@ -868,6 +869,10 @@ struct TEvBlobStorage {
         EvNodeWardenBaseConfig,
         EvNodeWardenDynamicConfigSubscribe,
         EvNodeWardenDynamicConfigPush,
+        EvNodeWardenReadMetadata,
+        EvNodeWardenReadMetadataResult,
+        EvNodeWardenWriteMetadata,
+        EvNodeWardenWriteMetadataResult,
 
         // Other
         EvRunActor = EvPut + 15 * 512,

ydb/core/blobstorage/nodewarden/distconf.cpp

@@ -34,9 +34,7 @@ namespace NKikimr::NStorage {
                 DrivesToRead.push_back(drive.GetPath());
             });
             std::sort(DrivesToRead.begin(), DrivesToRead.end());
-
-            auto query = std::bind(&TThis::ReadConfig, TActivationContext::ActorSystem(), SelfId(), DrivesToRead, Cfg, 0);
-            Send(MakeIoDispatcherActorId(), new TEvInvokeQuery(std::move(query)));
+            ReadConfig();
         } else {
             StorageConfigLoaded = true;
         }
@@ -181,6 +179,10 @@ namespace NKikimr::NStorage {
 #endif
 
     STFUNC(TDistributedConfigKeeper::StateWaitForInit) {
+        STLOG(PRI_DEBUG, BS_NODE, NWDC53, "StateWaitForInit event", (Type, ev->GetTypeRewrite()),
+            (StorageConfigLoaded, StorageConfigLoaded), (NodeListObtained, NodeListObtained),
+            (PendingEvents.size, PendingEvents.size()));
+
         auto processPendingEvents = [&] {
             if (PendingEvents.empty()) {
                 Become(&TThis::StateFunc);
@@ -195,7 +197,9 @@ namespace NKikimr::NStorage {
         switch (ev->GetTypeRewrite()) {
             case TEvInterconnect::TEvNodesInfo::EventType:
                 Handle(reinterpret_cast<TEvInterconnect::TEvNodesInfo::TPtr&>(ev));
-                change = !std::exchange(NodeListObtained, true);
+                if (!NodeIds.empty()) {
+                    change = !std::exchange(NodeListObtained, true);
+                }
                 break;
 
             case TEvPrivate::EvStorageConfigLoaded:
@@ -216,10 +220,6 @@ namespace NKikimr::NStorage {
         }
 
         if (change && NodeListObtained && StorageConfigLoaded) {
-            if (IsSelfStatic) {
-                UpdateBound(SelfNode.NodeId(), SelfNode, *StorageConfig, nullptr);
-                IssueNextBindRequest();
-            }
             processPendingEvents();
         }
     }

ydb/core/blobstorage/nodewarden/distconf.h

@@ -220,6 +220,7 @@ namespace NKikimr::NStorage {
         std::deque<TAutoPtr<IEventHandle>> PendingEvents;
         std::vector<ui32> NodeIds;
         TNodeIdentifier SelfNode;
+        bool SelfBound = false;
 
         // scatter tasks
         ui64 NextScatterCookie = RandomNumber<ui64>();
@@ -265,15 +266,10 @@ namespace NKikimr::NStorage {
         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
         // PDisk configuration retrieval and storing
 
-        static void ReadConfig(TActorSystem *actorSystem, TActorId selfId, const std::vector<TString>& drives,
-            const TIntrusivePtr<TNodeWardenConfig>& cfg, ui64 cookie);
-
-        static void WriteConfig(TActorSystem *actorSystem, TActorId selfId, const std::vector<TString>& drives,
-            const TIntrusivePtr<TNodeWardenConfig>& cfg, const NKikimrBlobStorage::TPDiskMetadataRecord& record);
-
+        void ReadConfig(ui64 cookie = 0);
+        void WriteConfig(std::vector<TString> drives, NKikimrBlobStorage::TPDiskMetadataRecord record);
         void PersistConfig(TPersistCallback callback);
         void Handle(TEvPrivate::TEvStorageConfigStored::TPtr ev);
-
         void Handle(TEvPrivate::TEvStorageConfigLoaded::TPtr ev);
 
         static TString CalculateFingerprint(const NKikimrBlobStorage::TStorageConfig& config);

ydb/core/blobstorage/nodewarden/distconf_binding.cpp

@@ -58,7 +58,8 @@ namespace NKikimr::NStorage {
         // issue updates
         NodeIds = std::move(nodeIds);
         BindQueue.Update(NodeIds);
-        if (NodeListObtained && StorageConfigLoaded) {
+        if (NodeListObtained && StorageConfigLoaded && !std::exchange(SelfBound, true)) {
+            UpdateBound(SelfNode.NodeId(), SelfNode, *StorageConfig, nullptr);
             IssueNextBindRequest();
         }
     }

ydb/core/blobstorage/nodewarden/distconf_fsm.cpp

@@ -328,8 +328,7 @@ namespace NKikimr::NStorage {
                 }
                 std::sort(drives.begin(), drives.end());
                 drives.erase(std::unique(drives.begin(), drives.end()), drives.end());
-                auto query = std::bind(&TThis::ReadConfig, TActivationContext::ActorSystem(), SelfId(), drives, Cfg, cookie);
-                Send(MakeIoDispatcherActorId(), new TEvInvokeQuery(std::move(query)));
+                ReadConfig(cookie);
                 ++task.AsyncOperationsPending;
                 break;
             }