Skip to content

Commit dbb2dc8

Browse files
alexvrualexvru
authored
Move distconf settings a level up in the configuration and add explicit enabled flag (#12696)
1 parent 881ce4a commit dbb2dc8

File tree

16 files changed

+163
-144
lines changed

16 files changed

+163
-144
lines changed

‎ydb/core/blobstorage/nodewarden/distconf.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ namespace NKikimr::NStorage {
339339

340340
struct TExConfigError : yexception {};
341341

342-
void GenerateFirstConfig(NKikimrBlobStorage::TStorageConfig *config, const TString& selfAssemblyUUID);
342+
std::optional<TString> GenerateFirstConfig(NKikimrBlobStorage::TStorageConfig *config, const TString& selfAssemblyUUID);
343343

344344
void AllocateStaticGroup(NKikimrBlobStorage::TStorageConfig *config, ui32 groupId, ui32 groupGeneration,
345345
TBlobStorageGroupType gtype, const NKikimrBlobStorage::TGroupGeometry& geometry,

‎ydb/core/blobstorage/nodewarden/distconf_fsm.cpp

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -349,13 +349,9 @@ namespace NKikimr::NStorage {
349349
configToPropose = persistedConfig;
350350
}
351351
} else if (baseConfig && !baseConfig->GetGeneration()) {
352-
bool canBootstrapAutomatically = false;
353-
if (baseConfig->HasBlobStorageConfig()) {
354-
if (const auto& bsConfig = baseConfig->GetBlobStorageConfig(); bsConfig.HasAutoconfigSettings()) {
355-
const auto& autoconfigSettings = bsConfig.GetAutoconfigSettings();
356-
canBootstrapAutomatically = autoconfigSettings.GetAutomaticBootstrap();
357-
}
358-
}
352+
const bool canBootstrapAutomatically = baseConfig->HasSelfManagementConfig() &&
353+
baseConfig->GetSelfManagementConfig().GetEnabled() &&
354+
baseConfig->GetSelfManagementConfig().GetAutomaticBootstrap();
359355
if (canBootstrapAutomatically || selfAssemblyUUID) {
360356
if (!selfAssemblyUUID) {
361357
if (!CurrentSelfAssemblyUUID) {
@@ -364,7 +360,9 @@ namespace NKikimr::NStorage {
364360
selfAssemblyUUID = &CurrentSelfAssemblyUUID.value();
365361
}
366362
propositionBase.emplace(*baseConfig);
367-
GenerateFirstConfig(baseConfig, *selfAssemblyUUID);
363+
if (auto error = GenerateFirstConfig(baseConfig, *selfAssemblyUUID)) {
364+
return *error;
365+
}
368366
configToPropose = baseConfig;
369367
}
370368
}

‎ydb/core/blobstorage/nodewarden/distconf_generate.cpp

Lines changed: 41 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,51 @@
66

77
namespace NKikimr::NStorage {
88

9-
void TDistributedConfigKeeper::GenerateFirstConfig(NKikimrBlobStorage::TStorageConfig *config, const TString& selfAssemblyUUID) {
10-
if (config->HasBlobStorageConfig()) {
11-
const auto& bsConfig = config->GetBlobStorageConfig();
12-
const bool noStaticGroup = !bsConfig.HasServiceSet() || !bsConfig.GetServiceSet().GroupsSize();
13-
if (noStaticGroup && bsConfig.HasAutoconfigSettings() && bsConfig.GetAutoconfigSettings().HasErasureSpecies()) {
14-
try {
15-
const auto& settings = bsConfig.GetAutoconfigSettings();
16-
17-
const auto species = TBlobStorageGroupType::ErasureSpeciesByName(settings.GetErasureSpecies());
18-
if (species == TBlobStorageGroupType::ErasureSpeciesCount) {
19-
throw TExConfigError() << "invalid erasure specified for static group"
20-
<< " Erasure# " << settings.GetErasureSpecies();
21-
}
9+
std::optional<TString> TDistributedConfigKeeper::GenerateFirstConfig(NKikimrBlobStorage::TStorageConfig *config,
10+
const TString& selfAssemblyUUID) {
11+
if (!config->HasSelfManagementConfig() || !config->GetSelfManagementConfig().GetEnabled()) {
12+
return "self-management is not enabled";
13+
}
14+
const auto& smConfig = config->GetSelfManagementConfig();
15+
16+
const bool noStaticGroup = !config->HasBlobStorageConfig() || // either no BlobStorageConfig section at all
17+
!config->GetBlobStorageConfig().HasServiceSet() || // or no ServiceSet in there
18+
!config->GetBlobStorageConfig().GetServiceSet().GroupsSize(); // or no groups in ServiceSet
19+
if (noStaticGroup) {
20+
try {
21+
if (!smConfig.HasErasureSpecies()) {
22+
return "missing ErasureSpecies in SelfManagementConfig";
23+
}
2224

23-
AllocateStaticGroup(config, 0 /*groupId*/, 1 /*groupGeneration*/, TBlobStorageGroupType(species),
24-
settings.GetGeometry(), settings.GetPDiskFilter(),
25-
settings.HasPDiskType() ? std::make_optional(settings.GetPDiskType()) : std::nullopt, {}, {}, 0,
26-
nullptr, false, true, false);
27-
STLOG(PRI_DEBUG, BS_NODE, NWDC33, "Allocated static group", (Group, bsConfig.GetServiceSet().GetGroups(0)));
28-
} catch (const TExConfigError& ex) {
29-
STLOG(PRI_ERROR, BS_NODE, NWDC10, "Failed to allocate static group", (Reason, ex.what()));
25+
const auto species = TBlobStorageGroupType::ErasureSpeciesByName(smConfig.GetErasureSpecies());
26+
if (species == TBlobStorageGroupType::ErasureSpeciesCount) {
27+
throw TExConfigError() << "invalid erasure specified for static group"
28+
<< " Erasure# " << smConfig.GetErasureSpecies();
3029
}
30+
31+
AllocateStaticGroup(config, 0 /*groupId*/, 1 /*groupGeneration*/, TBlobStorageGroupType(species),
32+
smConfig.GetGeometry(), smConfig.GetPDiskFilter(),
33+
smConfig.HasPDiskType() ? std::make_optional(smConfig.GetPDiskType()) : std::nullopt, {}, {}, 0,
34+
nullptr, false, true, false);
35+
STLOG(PRI_DEBUG, BS_NODE, NWDC33, "Allocated static group",
36+
(Group, config->GetBlobStorageConfig().GetServiceSet().GetGroups(0)));
37+
} catch (const TExConfigError& ex) {
38+
return TStringBuilder() << "failed to allocate static group: " << ex.what();
3139
}
3240
}
3341

42+
// initial config YAML is taken from the Cfg->SelfManagementConfig as it is cleared in TStorageConfig while
43+
// deriving it from NodeWarden configuration
44+
if (!Cfg->SelfManagementConfig || !Cfg->SelfManagementConfig->HasInitialConfigYaml()) {
45+
return "missing initial config YAML";
46+
}
47+
TStringStream ss;
48+
{
49+
TZstdCompress zstd(&ss);
50+
zstd << Cfg->SelfManagementConfig->GetInitialConfigYaml();
51+
}
52+
config->SetStorageConfigCompressedYAML(ss.Str());
53+
3454
if (!Cfg->DomainsConfig) { // no automatic configuration required
3555
} else if (Cfg->DomainsConfig->StateStorageSize() == 1) { // the StateStorage config is already defined explicitly, just migrate it
3656
const auto& ss = Cfg->DomainsConfig->GetStateStorage(0);
@@ -45,16 +65,7 @@ namespace NKikimr::NStorage {
4565

4666
config->SetSelfAssemblyUUID(selfAssemblyUUID);
4767

48-
if (const auto& bsconfig = Cfg->BlobStorageConfig; bsconfig.HasAutoconfigSettings()) {
49-
if (const auto& autoconfigSettings = bsconfig.GetAutoconfigSettings(); autoconfigSettings.HasInitialConfigYaml()) {
50-
TStringStream ss;
51-
{
52-
TZstdCompress zstd(&ss);
53-
zstd << autoconfigSettings.GetInitialConfigYaml();
54-
}
55-
config->SetStorageConfigCompressedYAML(ss.Str());
56-
}
57-
}
68+
return std::nullopt;
5869
}
5970

6071
void TDistributedConfigKeeper::AllocateStaticGroup(NKikimrBlobStorage::TStorageConfig *config, ui32 groupId,

‎ydb/core/blobstorage/nodewarden/distconf_invoke.cpp

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ namespace NKikimr::NStorage {
155155
return AdvanceGeneration();
156156

157157
case TQuery::kFetchStorageConfig:
158-
return FetchStorageConfig();
158+
return FetchStorageConfig(record.GetFetchStorageConfig().GetManual());
159159

160160
case TQuery::kReplaceStorageConfig:
161161
return ReplaceStorageConfig(record.GetReplaceStorageConfig().GetYAML());
@@ -373,10 +373,10 @@ namespace NKikimr::NStorage {
373373
}
374374
const auto& ss = bsConfig.GetServiceSet();
375375

376-
if (!bsConfig.HasAutoconfigSettings()) {
377-
return FinishWithError(TResult::ERROR, "no AutoconfigSettings defined");
376+
if (!config.HasSelfManagementConfig() || !config.GetSelfManagementConfig().GetEnabled()) {
377+
return FinishWithError(TResult::ERROR, "self-management is not enabled");
378378
}
379-
const auto& settings = bsConfig.GetAutoconfigSettings();
379+
const auto& smConfig = config.GetSelfManagementConfig();
380380

381381
THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks;
382382
NBsController::TGroupMapper::TForbiddenPDisks forbid;
@@ -405,8 +405,8 @@ namespace NKikimr::NStorage {
405405
try {
406406
Self->AllocateStaticGroup(&config, vdiskId.GroupID.GetRawId(), vdiskId.GroupGeneration + 1,
407407
TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)group.GetErasureSpecies()),
408-
settings.GetGeometry(), settings.GetPDiskFilter(),
409-
settings.HasPDiskType() ? std::make_optional(settings.GetPDiskType()) : std::nullopt,
408+
smConfig.GetGeometry(), smConfig.GetPDiskFilter(),
409+
smConfig.HasPDiskType() ? std::make_optional(smConfig.GetPDiskType()) : std::nullopt,
410410
replacedDisks, forbid, maxSlotSize,
411411
&BaseConfig.value(), cmd.GetConvertToDonor(), cmd.GetIgnoreVSlotQuotaCheck(),
412412
cmd.GetIsSelfHealReasonDecommit());
@@ -625,7 +625,7 @@ namespace NKikimr::NStorage {
625625
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
626626
// Storage configuration YAML manipulation
627627

628-
void FetchStorageConfig() {
628+
void FetchStorageConfig(bool manual) {
629629
if (!Self->StorageConfig) {
630630
FinishWithError(TResult::ERROR, "no agreed StorageConfig");
631631
} else if (!Self->StorageConfig->HasStorageConfigCompressedYAML()) {
@@ -634,7 +634,13 @@ namespace NKikimr::NStorage {
634634
auto ev = PrepareResult(TResult::OK, std::nullopt);
635635
auto *record = &ev->Record;
636636
TStringInput ss(Self->StorageConfig->GetStorageConfigCompressedYAML());
637-
record->MutableFetchStorageConfig()->SetYAML(TZstdDecompress(&ss).ReadAll());
637+
TString config = TZstdDecompress(&ss).ReadAll();
638+
639+
if (manual) {
640+
// add BlobStorageConfig, NameserviceConfig, DomainsConfig
641+
}
642+
643+
record->MutableFetchStorageConfig()->SetYAML(config);
638644
Finish(Sender, SelfId(), ev.release(), 0, Cookie);
639645
}
640646
}

‎ydb/core/blobstorage/nodewarden/node_warden.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ namespace NKikimr {
2121
NKikimrConfig::TBlobStorageConfig BlobStorageConfig;
2222
NKikimrConfig::TStaticNameserviceConfig NameserviceConfig;
2323
std::optional<NKikimrConfig::TDomainsConfig> DomainsConfig;
24+
std::optional<NKikimrConfig::TSelfManagementConfig> SelfManagementConfig;
2425
TIntrusivePtr<IPDiskServiceFactory> PDiskServiceFactory;
2526
TIntrusivePtr<TAllVDiskKinds> AllVDiskKinds;
2627
TIntrusivePtr<NPDisk::TDriveModelDb> AllDriveModels;

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -392,11 +392,14 @@ void TNodeWarden::Bootstrap() {
392392

393393
// fill in a base storage config (from the file)
394394
NKikimrConfig::TAppConfig appConfig;
395+
appConfig.MutableBlobStorageConfig()->CopyFrom(Cfg->BlobStorageConfig);
396+
appConfig.MutableNameserviceConfig()->CopyFrom(Cfg->NameserviceConfig);
395397
if (Cfg->DomainsConfig) {
396398
appConfig.MutableDomainsConfig()->CopyFrom(*Cfg->DomainsConfig);
397399
}
398-
appConfig.MutableBlobStorageConfig()->CopyFrom(Cfg->BlobStorageConfig);
399-
appConfig.MutableNameserviceConfig()->CopyFrom(Cfg->NameserviceConfig);
400+
if (Cfg->SelfManagementConfig) {
401+
appConfig.MutableSelfManagementConfig()->CopyFrom(*Cfg->SelfManagementConfig);
402+
}
400403
TString errorReason;
401404
const bool success = DeriveStorageConfig(appConfig, &StorageConfig, &errorReason);
402405
Y_VERIFY_S(success, "failed to generate initial TStorageConfig: " << errorReason);
@@ -992,26 +995,23 @@ bool NKikimr::NStorage::DeriveStorageConfig(const NKikimrConfig::TAppConfig& app
992995
*errorReason = "original config missing mandatory BlobStorageConfig section";
993996
return false;
994997
}
995-
996-
const auto& bsFrom = appConfig.GetBlobStorageConfig();
997-
auto *bsTo = config->MutableBlobStorageConfig();
998-
if (bsFrom.HasAutoconfigSettings()) {
999-
const auto& acFrom = bsFrom.GetAutoconfigSettings();
1000-
auto *acTo = bsTo->MutableAutoconfigSettings();
1001-
if (acFrom.HasGeneration() && acTo->HasGeneration() && acTo->GetGeneration() + 1 != acFrom.GetGeneration()) {
1002-
*errorReason = TStringBuilder() << "generation mismatch for AutoconfigSettings section existing Generation# "
1003-
<< acTo->GetGeneration() << " newly provided Generation# " << acFrom.GetGeneration();
1004-
return false;
1005-
} else if (acTo->HasGeneration() && !acFrom.HasGeneration()) {
1006-
*errorReason = "existing AutoconfigSettings has set generation, but newly provided one doesn't have it";
998+
999+
if (appConfig.HasSelfManagementConfig()) {
1000+
const auto& smFrom = appConfig.GetSelfManagementConfig();
1001+
auto *smTo = config->MutableSelfManagementConfig();
1002+
if (smFrom.HasGeneration() && smTo->HasGeneration() && smFrom.GetGeneration() != smTo->GetGeneration() + 1) {
1003+
*errorReason = TStringBuilder() << "generation mismatch for SelfManagementConfig section existing Generation# "
1004+
<< smTo->GetGeneration() << " newly provided Generation# " << smFrom.GetGeneration();
10071005
return false;
10081006
}
1009-
1010-
acTo->CopyFrom(acFrom);
1011-
acTo->ClearInitialConfigYaml();
1007+
smTo->CopyFrom(smFrom);
1008+
smTo->ClearInitialConfigYaml(); // do not let this section into final StorageConfig
10121009
} else {
1013-
bsTo->ClearAutoconfigSettings();
1010+
config->ClearSelfManagementConfig();
10141011
}
1012+
1013+
const auto& bsFrom = appConfig.GetBlobStorageConfig();
1014+
auto *bsTo = config->MutableBlobStorageConfig();
10151015

10161016
if (bsFrom.HasServiceSet()) {
10171017
const auto& ssFrom = bsFrom.GetServiceSet();
@@ -1034,7 +1034,7 @@ bool NKikimr::NStorage::DeriveStorageConfig(const NKikimrConfig::TAppConfig& app
10341034
};
10351035

10361036
// update static group information unless distconf is enabled
1037-
if (!hasStaticGroupInfo(ssFrom) && bsFrom.HasAutoconfigSettings()) {
1037+
if (!hasStaticGroupInfo(ssFrom) && config->HasSelfManagementConfig() && config->GetSelfManagementConfig().GetEnabled()) {
10381038
// distconf enabled, keep it as is
10391039
} else if (!hasStaticGroupInfo(*ssTo)) {
10401040
ssTo->MutablePDisks()->CopyFrom(ssFrom.GetPDisks());

‎ydb/core/config/init/init.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -589,9 +589,9 @@ void LoadYamlConfig(TConfigRefs refs, const TString& yamlConfigFile, NKikimrConf
589589

590590
const TString yamlConfigString = protoConfigFileProvider.GetProtoFromFile(yamlConfigFile, errorCollector);
591591

592-
if (appConfig.HasBlobStorageConfig() && appConfig.GetBlobStorageConfig().HasAutoconfigSettings()) {
593-
auto *proto = appConfig.MutableBlobStorageConfig()->MutableAutoconfigSettings();
594-
proto->SetInitialConfigYaml(yamlConfigString);
592+
if (appConfig.HasSelfManagementConfig() && appConfig.GetSelfManagementConfig().GetEnabled()) {
593+
// fill in InitialConfigYaml only when self-management through distconf is enabled
594+
appConfig.MutableSelfManagementConfig()->SetInitialConfigYaml(yamlConfigString);
595595
}
596596

597597
/*

‎ydb/core/driver_lib/run/kikimr_services_initializers.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -945,6 +945,9 @@ void TBSNodeWardenInitializer::InitializeServices(NActors::TActorSystemSetup* se
945945
if (Config.HasDomainsConfig()) {
946946
nodeWardenConfig->DomainsConfig.emplace(Config.GetDomainsConfig());
947947
}
948+
if (Config.HasSelfManagementConfig()) {
949+
nodeWardenConfig->SelfManagementConfig.emplace(Config.GetSelfManagementConfig());
950+
}
948951

949952
ObtainTenantKey(&nodeWardenConfig->TenantKey, Config.GetKeyConfig());
950953
ObtainStaticKey(&nodeWardenConfig->StaticKey);

‎ydb/core/grpc_services/rpc_bsconfig.cpp

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,11 +109,7 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc<TReplaceStorage
109109
} catch (const std::exception&) {
110110
return false; // assuming no distconf enabled in this config
111111
}
112-
if (!newConfig.HasBlobStorageConfig()) {
113-
return false;
114-
}
115-
const NKikimrConfig::TBlobStorageConfig& bsConfig = newConfig.GetBlobStorageConfig();
116-
return bsConfig.HasAutoconfigSettings();
112+
return newConfig.HasSelfManagementConfig() && newConfig.GetSelfManagementConfig().GetEnabled();
117113
}
118114
};
119115

‎ydb/core/mind/bscontroller/bsc.cpp

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -155,14 +155,14 @@ void TBlobStorageController::Handle(TEvNodeWardenStorageConfig::TPtr ev) {
155155
} else {
156156
Y_FAIL("no storage configuration provided");
157157
}
158+
}
158159

159-
if (bsConfig.HasAutoconfigSettings()) {
160-
// assuming that in autoconfig mode HostRecords are managed by the distconf; we need to apply it here to
161-
// avoid race with box autoconfiguration and node list change
162-
HostRecords = std::make_shared<THostRecordMap::element_type>(StorageConfig);
163-
if (SelfHealId) {
164-
Send(SelfHealId, new TEvPrivate::TEvUpdateHostRecords(HostRecords));
165-
}
160+
if (StorageConfig.HasSelfManagementConfig() && StorageConfig.GetSelfManagementConfig().GetEnabled()) {
161+
// assuming that in autoconfig mode HostRecords are managed by the distconf; we need to apply it here to
162+
// avoid race with box autoconfiguration and node list change
163+
HostRecords = std::make_shared<THostRecordMap::element_type>(StorageConfig);
164+
if (SelfHealId) {
165+
Send(SelfHealId, new TEvPrivate::TEvUpdateHostRecords(HostRecords));
166166
}
167167
}
168168

@@ -188,20 +188,14 @@ void TBlobStorageController::Handle(TEvents::TEvUndelivered::TPtr ev) {
188188
}
189189

190190
void TBlobStorageController::ApplyStorageConfig() {
191-
if (!StorageConfig.HasBlobStorageConfig()) {
191+
if (!StorageConfig.HasBlobStorageConfig() || // this would be strange
192+
!StorageConfig.HasSelfManagementConfig() ||
193+
!StorageConfig.GetSelfManagementConfig().GetEnabled() ||
194+
!StorageConfig.GetSelfManagementConfig().GetAutomaticBoxManagement()) {
192195
return;
193196
}
194197
const auto& bsConfig = StorageConfig.GetBlobStorageConfig();
195198

196-
if (!bsConfig.HasAutoconfigSettings()) {
197-
return;
198-
}
199-
const auto& autoconfigSettings = bsConfig.GetAutoconfigSettings();
200-
201-
if (autoconfigSettings.HasAutomaticBoxManagement() && !autoconfigSettings.GetAutomaticBoxManagement()) {
202-
return;
203-
}
204-
205199
if (Boxes.size() > 1) {
206200
return;
207201
}

0 commit comments

Comments
 (0)