Skip to content

Commit ee3d7ef

Browse files
alexvrualexvru
authored
Public API bootstrap command (#12308)
1 parent e73e490 commit ee3d7ef

File tree

22 files changed

+246
-18
lines changed

22 files changed

+246
-18
lines changed

‎ydb/apps/ydb/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* Made `--consumer` flag in `ydb topic read` command optional. Now if this flag is not specified, reading is performed in no-consumer mode. In this mode partition IDs should be specified with `--partition-ids` option.
44
* Fixed a bug in `ydb import file csv` where multiple columns with escaped quotes in the same row were parsed incorrectly
55
* Truncate query results output in benchmarks
6+
* Added `ydb admin cluster bootstrap` command to bootstrap automatically configured cluster
67

78
## 2.17.0 ##
89

‎ydb/core/blobstorage/nodewarden/distconf_generate.cpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
#include <ydb/core/mind/bscontroller/group_geometry_info.h>
44

5+
#include <library/cpp/streams/zstd/zstd.h>
6+
57
namespace NKikimr::NStorage {
68

79
void TDistributedConfigKeeper::GenerateFirstConfig(NKikimrBlobStorage::TStorageConfig *config, const TString& selfAssemblyUUID) {
@@ -42,6 +44,17 @@ namespace NKikimr::NStorage {
4244
}
4345

4446
config->SetSelfAssemblyUUID(selfAssemblyUUID);
47+
48+
if (const auto& bsconfig = Cfg->BlobStorageConfig; bsconfig.HasAutoconfigSettings()) {
49+
if (const auto& autoconfigSettings = bsconfig.GetAutoconfigSettings(); autoconfigSettings.HasInitialConfigYaml()) {
50+
TStringStream ss;
51+
{
52+
TZstdCompress zstd(&ss);
53+
zstd << autoconfigSettings.GetInitialConfigYaml();
54+
}
55+
config->SetStorageConfigCompressedYAML(ss.Str());
56+
}
57+
}
4558
}
4659

4760
void TDistributedConfigKeeper::AllocateStaticGroup(NKikimrBlobStorage::TStorageConfig *config, ui32 groupId,

‎ydb/core/blobstorage/nodewarden/distconf_invoke.cpp

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -160,8 +160,8 @@ namespace NKikimr::NStorage {
160160
case TQuery::kReplaceStorageConfig:
161161
return ReplaceStorageConfig(record.GetReplaceStorageConfig().GetYAML());
162162

163-
case TQuery::kBootstrap:
164-
return DoBootstrap(record.GetBootstrap().GetSelfAssemblyUUID());
163+
case TQuery::kBootstrapCluster:
164+
return BootstrapCluster(record.GetBootstrapCluster().GetSelfAssemblyUUID());
165165

166166
case TQuery::REQUEST_NOT_SET:
167167
return FinishWithError(TResult::ERROR, "Request field not set");
@@ -670,7 +670,7 @@ namespace NKikimr::NStorage {
670670
StartProposition(&config);
671671
}
672672

673-
void DoBootstrap(const TString& selfAssemblyUUID) {
673+
void BootstrapCluster(const TString& selfAssemblyUUID) {
674674
if (!RunCommonChecks()) {
675675
return;
676676
} else if (Self->StorageConfig->GetGeneration()) {
@@ -679,6 +679,8 @@ namespace NKikimr::NStorage {
679679
} else {
680680
return FinishWithError(TResult::ERROR, "bootstrap on already bootstrapped cluster");
681681
}
682+
} else if (!selfAssemblyUUID) {
683+
return FinishWithError(TResult::ERROR, "SelfAssemblyUUID can't be empty");
682684
}
683685

684686
const ERootState prevState = std::exchange(Self->RootState, ERootState::IN_PROGRESS);
@@ -765,6 +767,8 @@ namespace NKikimr::NStorage {
765767
auto *propose = task.MutableProposeStorageConfig();
766768
propose->MutableConfig()->CopyFrom(*Self->CurrentProposedStorageConfig);
767769
IssueScatterTask(std::move(task), done);
770+
771+
Self->RootState = ERootState::IN_PROGRESS; // forbid any concurrent activity
768772
}
769773

770774
////////////////////////////////////////////////////////////////////////////////////////////////////////////////

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1005,6 +1005,7 @@ bool NKikimr::NStorage::DeriveStorageConfig(const NKikimrConfig::TAppConfig& app
10051005
}
10061006

10071007
acTo->CopyFrom(acFrom);
1008+
acTo->ClearInitialConfigYaml();
10081009
} else {
10091010
bsTo->ClearAutoconfigSettings();
10101011
}

‎ydb/core/config/init/init.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -588,6 +588,12 @@ void LoadYamlConfig(TConfigRefs refs, const TString& yamlConfigFile, NKikimrConf
588588
IProtoConfigFileProvider& protoConfigFileProvider = refs.ProtoConfigFileProvider;
589589

590590
const TString yamlConfigString = protoConfigFileProvider.GetProtoFromFile(yamlConfigFile, errorCollector);
591+
592+
if (appConfig.HasBlobStorageConfig() && appConfig.GetBlobStorageConfig().HasAutoconfigSettings()) {
593+
auto *proto = appConfig.MutableBlobStorageConfig()->MutableAutoconfigSettings();
594+
proto->SetInitialConfigYaml(yamlConfigString);
595+
}
596+
591597
/*
592598
* FIXME: if (ErrorCollector.HasFatal()) { return; }
593599
*/

‎ydb/core/grpc_services/rpc_bsconfig.cpp

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <ydb/core/grpc_services/rpc_common/rpc_common.h>
88
#include <ydb/core/mind/local.h>
99
#include <ydb/core/protos/local.pb.h>
10+
#include <ydb/core/blobstorage/nodewarden/node_warden_events.h>
1011

1112
namespace NKikimr::NGRpcService {
1213

@@ -16,6 +17,9 @@ using TEvReplaceStorageConfigRequest =
1617
using TEvFetchStorageConfigRequest =
1718
TGrpcRequestOperationCall<Ydb::BSConfig::FetchStorageConfigRequest,
1819
Ydb::BSConfig::FetchStorageConfigResponse>;
20+
using TEvBootstrapClusterRequest =
21+
TGrpcRequestOperationCall<Ydb::BSConfig::BootstrapClusterRequest,
22+
Ydb::BSConfig::BootstrapClusterResponse>;
1923

2024
using namespace NActors;
2125
using namespace Ydb;
@@ -88,6 +92,15 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc<TReplaceStorage
8892
NACLib::EAccessRights GetRequiredAccessRights() const {
8993
return NACLib::GenericManage;
9094
}
95+
96+
void FillDistconfQuery(NStorage::TEvNodeConfigInvokeOnRoot& ev) {
97+
auto *cmd = ev.Record.MutableReplaceStorageConfig();
98+
cmd->SetYAML(GetProtoRequest()->yaml_config());
99+
}
100+
101+
void FillDistconfResult(NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult& /*record*/,
102+
Ydb::BSConfig::ReplaceStorageConfigResult& /*result*/)
103+
{}
91104
};
92105

93106
class TFetchStorageConfigRequest : public TBSConfigRequestGrpc<TFetchStorageConfigRequest, TEvFetchStorageConfigRequest,
@@ -102,6 +115,15 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpc<TFetchStorageConf
102115
NACLib::EAccessRights GetRequiredAccessRights() const {
103116
return NACLib::GenericManage;
104117
}
118+
119+
void FillDistconfQuery(NStorage::TEvNodeConfigInvokeOnRoot& ev) const {
120+
ev.Record.MutableFetchStorageConfig();
121+
}
122+
123+
void FillDistconfResult(NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult& record,
124+
Ydb::BSConfig::FetchStorageConfigResult& result) {
125+
result.set_yaml_config(record.GetFetchStorageConfig().GetYAML());
126+
}
105127
};
106128

107129
void DoReplaceBSConfig(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
@@ -112,5 +134,50 @@ void DoFetchBSConfig(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&)
112134
TActivationContext::AsActorContext().Register(new TFetchStorageConfigRequest(p.release()));
113135
}
114136

137+
void DoBootstrapCluster(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
138+
class TBootstrapClusterRequest : public TRpcOperationRequestActor<TBootstrapClusterRequest, TEvBootstrapClusterRequest> {
139+
using TBase = TRpcOperationRequestActor<TBootstrapClusterRequest, TEvBootstrapClusterRequest>;
140+
141+
public:
142+
using TBase::TBase;
143+
144+
void Bootstrap(const TActorContext& ctx) {
145+
TBase::Bootstrap(ctx);
146+
Become(&TBootstrapClusterRequest::StateFunc);
147+
148+
const auto& request = *GetProtoRequest();
149+
150+
auto ev = std::make_unique<NStorage::TEvNodeConfigInvokeOnRoot>();
151+
auto& record = ev->Record;
152+
auto *cmd = record.MutableBootstrapCluster();
153+
cmd->SetSelfAssemblyUUID(request.self_assembly_uuid());
154+
Send(MakeBlobStorageNodeWardenID(SelfId().NodeId()), ev.release());
155+
}
156+
157+
void Handle(NStorage::TEvNodeConfigInvokeOnRootResult::TPtr ev, const TActorContext& ctx) {
158+
auto& record = ev->Get()->Record;
159+
switch (record.GetStatus()) {
160+
case NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult::OK:
161+
Reply(Ydb::StatusIds::SUCCESS, ctx);
162+
break;
163+
164+
default:
165+
Reply(Ydb::StatusIds::GENERIC_ERROR, record.GetErrorReason(), NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx);
166+
break;
167+
}
168+
}
169+
170+
protected:
171+
STFUNC(StateFunc) {
172+
switch (ev->GetTypeRewrite()) {
173+
HFunc(NStorage::TEvNodeConfigInvokeOnRootResult, Handle);
174+
default:
175+
return TBase::StateFuncBase(ev);
176+
}
177+
}
178+
};
179+
180+
TActivationContext::Register(new TBootstrapClusterRequest(p.release()));
181+
}
115182

116-
} // namespace NKikimr::NGRpcService
183+
} // namespace NKikimr::NGRpcService

‎ydb/core/grpc_services/rpc_bsconfig_base.h

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
#include <ydb/public/api/protos/ydb_bsconfig.pb.h>
77
#include <ydb/core/blobstorage/base/blobstorage_events.h>
8+
#include <ydb/core/blobstorage/nodewarden/node_warden_events.h>
89
#include <ydb/core/protos/blobstorage_base3.pb.h>
910
#include <ydb/core/base/tabletid.h>
1011
#include <ydb/library/ydb_issue/issue_helpers.h>
@@ -155,11 +156,11 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor<TDerived, TRequest
155156

156157
void Bootstrap(const TActorContext &ctx) {
157158
TBase::Bootstrap(ctx);
158-
this->OnBootstrap();
159-
this->Become(&TBSConfigRequestGrpc::StateFunc);
160-
BSCTabletId = MakeBSControllerID();
161-
CreatePipe();
162-
SendRequest();
159+
auto *self = Self();
160+
self->OnBootstrap();
161+
self->Become(&TBSConfigRequestGrpc::StateFunc);
162+
// ask Node Warden for configuration to see whether we are using distconf
163+
self->Send(MakeBlobStorageNodeWardenID(ctx.SelfID.NodeId()), new TEvNodeWardenQueryStorageConfig(false));
163164
}
164165

165166
protected:
@@ -168,11 +169,39 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor<TDerived, TRequest
168169
hFunc(TEvTabletPipe::TEvClientConnected, Handle);
169170
hFunc(TEvTabletPipe::TEvClientDestroyed, Handle);
170171
hFunc(TEvBlobStorage::TEvControllerConfigResponse, Handle);
172+
hFunc(TEvNodeWardenStorageConfig, Handle);
173+
hFunc(NStorage::TEvNodeConfigInvokeOnRootResult, Handle);
171174
default:
172175
return TBase::StateFuncBase(ev);
173176
}
174177
}
175178

179+
void Handle(TEvNodeWardenStorageConfig::TPtr ev) {
180+
auto *self = Self();
181+
if (ev->Get()->Config->GetGeneration()) { // distconf
182+
auto ev = std::make_unique<NStorage::TEvNodeConfigInvokeOnRoot>();
183+
self->FillDistconfQuery(*ev);
184+
self->Send(MakeBlobStorageNodeWardenID(self->SelfId().NodeId()), ev.release());
185+
} else { // classic BSC
186+
BSCTabletId = MakeBSControllerID();
187+
CreatePipe();
188+
SendRequest();
189+
}
190+
}
191+
192+
void Handle(NStorage::TEvNodeConfigInvokeOnRootResult::TPtr ev) {
193+
auto *self = Self();
194+
auto& record = ev->Get()->Record;
195+
if (record.GetStatus() != NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult::OK) {
196+
self->Reply(Ydb::StatusIds::INTERNAL_ERROR, record.GetErrorReason(), NKikimrIssues::TIssuesIds::DEFAULT_ERROR,
197+
TActivationContext::AsActorContext());
198+
} else {
199+
TResultRecord result;
200+
self->FillDistconfResult(record, result);
201+
self->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, TActivationContext::AsActorContext());
202+
}
203+
}
204+
176205
NTabletPipe::TClientConfig GetPipeConfig() {
177206
NTabletPipe::TClientConfig cfg;
178207
cfg.RetryPolicy = {
@@ -219,6 +248,10 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor<TDerived, TRequest
219248
}
220249

221250
virtual bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) = 0;
251+
252+
const TDerived *Self() const { return static_cast<const TDerived*>(this); }
253+
TDerived *Self() { return static_cast<TDerived*>(this); }
254+
222255
private:
223256
ui64 BSCTabletId = 0;
224257
TActorId BSCPipeClient;

‎ydb/core/grpc_services/service_bsconfig.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,6 @@ namespace NKikimr::NGRpcService {
1111

1212
void DoFetchBSConfig(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&);
1313

14-
} // NKikimr::NGRpcService
14+
void DoBootstrapCluster(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&);
15+
16+
} // NKikimr::NGRpcService

‎ydb/core/jaeger_tracing/request_discriminator.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ enum class ERequestType: size_t {
7070

7171
BSCONFIG_REPLACESTORAGECONFIG,
7272
BSCONFIG_FETCHSTORAGECONFIG,
73+
BSCONFIG_BOOTSTRAP,
7374

7475
PING_GRPC,
7576
PING_PROXY,
@@ -141,6 +142,7 @@ static const THashMap<TStringBuf, ERequestType> NameToRequestType = {
141142

142143
{"BSConfig.ReplaceStorageConfig", ERequestType::BSCONFIG_REPLACESTORAGECONFIG},
143144
{"BSConfig.FetchStorageConfig", ERequestType::BSCONFIG_FETCHSTORAGECONFIG},
145+
{"BSConfig.Bootstrap", ERequestType::BSCONFIG_BOOTSTRAP},
144146
};
145147

146148
struct TRequestDiscriminator {

‎ydb/core/protos/blobstorage_distributed_config.proto

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ message TEvNodeConfigInvokeOnRoot {
195195
string YAML = 1;
196196
}
197197

198-
message TBootstrap {
198+
message TBootstrapCluster {
199199
string SelfAssemblyUUID = 1;
200200
}
201201

@@ -209,7 +209,7 @@ message TEvNodeConfigInvokeOnRoot {
209209
TAdvanceGeneration AdvanceGeneration = 7;
210210
TFetchStorageConfig FetchStorageConfig = 8;
211211
TReplaceStorageConfig ReplaceStorageConfig = 9;
212-
TBootstrap Bootstrap = 10;
212+
TBootstrapCluster BootstrapCluster = 10;
213213
}
214214
}
215215

‎ydb/core/protos/config.proto

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,9 @@ message TBlobStorageConfig {
295295

296296
// generation of the distconf section; when set, one can automatically apply in-filesystem config
297297
optional uint64 Generation = 8;
298+
299+
// internally-used field with initial config.yaml loaded on the start
300+
optional bytes InitialConfigYaml = 10;
298301
}
299302

300303
optional TAutoconfigSettings AutoconfigSettings = 6;

‎ydb/public/api/grpc/ydb_bsconfig_v1.proto

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@ import "ydb/public/api/protos/ydb_bsconfig.proto";
1010

1111
service BSConfigService {
1212

13-
// Initialize Blobstorage host configs and box
13+
// Initialize Blobstorage/single config
1414
rpc ReplaceStorageConfig(BSConfig.ReplaceStorageConfigRequest) returns (BSConfig.ReplaceStorageConfigResponse);
1515

16-
// Fetch Blobstorage host configs and box
16+
// Fetch Blobstorage/single config
1717
rpc FetchStorageConfig(BSConfig.FetchStorageConfigRequest) returns (BSConfig.FetchStorageConfigResponse);
1818

19-
}
19+
// Bootstrap automatically configured cluster
20+
rpc BootstrapCluster(BSConfig.BootstrapClusterRequest) returns (BSConfig.BootstrapClusterResponse);
21+
22+
}

‎ydb/public/api/protos/ydb_bsconfig.proto

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,13 @@ message FetchStorageConfigResponse {
3535

3636
message FetchStorageConfigResult {
3737
string yaml_config = 1;
38-
}
38+
}
39+
40+
message BootstrapClusterRequest {
41+
Ydb.Operations.OperationParams operation_params = 1;
42+
string self_assembly_uuid = 2;
43+
}
44+
45+
message BootstrapClusterResponse {
46+
Ydb.Operations.Operation operation = 1;
47+
}

‎ydb/public/lib/ydb_cli/commands/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ SRCS(
99
query_workload.cpp
1010
ydb_admin.cpp
1111
ydb_benchmark.cpp
12+
ydb_cluster.cpp
1213
ydb_debug.cpp
1314
ydb_dynamic_config.cpp
1415
ydb_ping.cpp

‎ydb/public/lib/ydb_cli/commands/ydb_admin.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include "ydb_dynamic_config.h"
44
#include "ydb_storage_config.h"
5+
#include "ydb_cluster.h"
56

67
namespace NYdb {
78
namespace NConsoleClient {
@@ -12,6 +13,7 @@ TCommandAdmin::TCommandAdmin()
1213
AddCommand(std::make_unique<NDynamicConfig::TCommandConfig>());
1314
AddCommand(std::make_unique<NDynamicConfig::TCommandVolatileConfig>());
1415
AddCommand(std::make_unique<NStorageConfig::TCommandStorageConfig>());
16+
AddCommand(std::make_unique<NCluster::TCommandCluster>());
1517
}
1618

1719
}

0 commit comments

Comments
 (0)