.

Author: SammyVimes

ya.conf

@@ -4,7 +4,7 @@ build_cache = true
 build_cache_conf = ['cas_logging=true', 'graph_info=true']
 build_cache_master = true
 cache_codec = ''
-cache_size = 150374182400
+cache_size = 300748364800
 content_uids = true
 dir_outputs = true
 dir_outputs_test_mode = true

ydb/apps/dstool/lib/dstool_cmd_cluster_workload_run.py

@@ -18,6 +18,7 @@ def add_options(p):
     p.add_argument('--enable-pdisk-encryption-keys-changes', action='store_true', help='Enable changes of PDisk encryption keys')
     p.add_argument('--enable-kill-tablets', action='store_true', help='Enable tablet killer')
     p.add_argument('--enable-kill-blob-depot', action='store_true', help='Enable BlobDepot killer')
+    p.add_argument('--enable-restart-pdisks', action='store_true', help='Enable PDisk restarter')
     p.add_argument('--kill-signal', type=str, default='KILL', help='Kill signal to send to restart node')
 
 
@@ -144,6 +145,19 @@ def do_restart(node_id):
             if args.enable_pdisk_encryption_keys_changes:
                 remove_old_pdisk_keys(pdisk_keys, pdisk_key_versions, node_id)
 
+        def do_restart_pdisk(node_id, pdisk_id):
+            assert can_act_on_vslot(node_id, pdisk_id)
+            request = common.kikimr_bsconfig.TConfigRequest(IgnoreDegradedGroupsChecks=True)
+            cmd = request.Command.add().RestartPDisk
+            cmd.TargetPDiskId.NodeId = node_id
+            cmd.TargetPDiskId.PDiskId = pdisk_id
+            try:
+                response = common.invoke_bsc_request(request)
+            except Exception as e:
+                raise Exception('failed to perform restart request: %s' % e)
+            if not response.Success:
+                raise Exception('Unexpected error from BSC: %s' % response.ErrorDescription)
+
         def do_evict(vslot_id):
             assert can_act_on_vslot(*vslot_id)
             try:
@@ -230,13 +244,16 @@ def do_kill_blob_depot():
         wipes = []
         readonlies = []
         unreadonlies = []
+        pdisk_restarts = []
 
         for vslot in base_config.VSlot:
             if common.is_dynamic_group(vslot.GroupId):
                 vslot_id = common.get_vslot_id(vslot.VSlotId)
                 vdisk_id = '[%08x:%d:%d:%d]' % (vslot.GroupId, vslot.FailRealmIdx, vslot.FailDomainIdx, vslot.VDiskIdx)
                 if vslot_id in vslot_readonly and not args.disable_readonly:
                     unreadonlies.append(('un-readonly vslot id: %s, vdisk id: %s' % (vslot_id, vdisk_id), (do_readonly, vslot, False)))
+                if can_act_on_vslot(*vslot_id[:2]) and args.enable_restart_pdisks:
+                    pdisk_restarts.append(('restart pdisk node_id: %d, pdisk_id: %d' % vslot_id[:2], (do_restart_pdisk, *vslot_id[:2])))
                 if can_act_on_vslot(*vslot_id) and (recent_restarts or args.disable_restarts):
                     if not args.disable_evicts:
                         evicts.append(('evict vslot id: %s, vdisk id: %s' % (vslot_id, vdisk_id), (do_evict, vslot_id)))
@@ -258,6 +275,8 @@ def pick(v):
             possible_actions.append(('readonly', (pick, readonlies)))
         if unreadonlies:
             possible_actions.append(('un-readonly', (pick, unreadonlies)))
+        if pdisk_restarts:
+            possible_actions.append(('restart-pdisk', (pick, pdisk_restarts)))
 
         restarts = []
 

ydb/apps/dstool/lib/dstool_cmd_group_virtual_create.py

@@ -67,7 +67,7 @@ def do(args):
                         names_remaining.remove(group.VirtualGroupInfo.Name)
                     elif group.VirtualGroupInfo.State == common.EVirtualGroupState.CREATE_FAILED:
                         names_remaining.remove(group.VirtualGroupInfo.Name)
-                        errors.append(f'{group.VirtualGroupInfo.Name}: {group.ErrorReason}')
+                        errors.append(f'{group.VirtualGroupInfo.Name}: {group.VirtualGroupInfo.ErrorReason}')
 
             if names_remaining:
                 time.sleep(1)

ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp

@@ -273,23 +273,30 @@ namespace NKikimr::NStorage {
     void TNodeWarden::DoRestartLocalPDisk(const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk& pdisk) {
         ui32 pdiskId = pdisk.GetPDiskID();
 
-        const TActorId actorId = MakeBlobStoragePDiskID(LocalNodeId, pdiskId);
+        STLOG(PRI_NOTICE, BS_NODE, NW75, "DoRestartLocalPDisk", (PDiskId, pdiskId));
+
+        const auto [_, inserted] = PDiskRestartInFlight.emplace(pdiskId);
+
+        if (!inserted) {
+            STLOG(PRI_NOTICE, BS_NODE, NW76, "Restart already in progress", (PDiskId, pdiskId));
+            // Restart is already in progress.
+            return;
+        }
 
         auto it = LocalPDisks.find(TPDiskKey(LocalNodeId, pdiskId));
         if (it == LocalPDisks.end()) {
+            PDiskRestartInFlight.erase(pdiskId);
+
+            STLOG(PRI_NOTICE, BS_NODE, NW77, "Restart state carried from previous start, just starting", (PDiskId, pdiskId));
+
             // This can happen if warden didn't handle pdisk's restart before node's restart.
             // In this case, PDisk has EntityStatus::RESTART instead of EntityStatus::INITIAL.
             StartLocalPDisk(pdisk);
             SendPDiskReport(pdiskId, NKikimrBlobStorage::TEvControllerNodeReport::PD_RESTARTED);
             return;
         }
 
-        const auto [_, inserted] = PDiskRestartInFlight.emplace(pdiskId);
-
-        if (!inserted) {
-            // Restart is already in progress.
-            return;
-        }
+        const TActorId actorId = MakeBlobStoragePDiskID(LocalNodeId, pdiskId);
 
         TIntrusivePtr<TPDiskConfig> pdiskConfig = CreatePDiskConfig(it->second.Record);