Skip to content

Commit 7df7f9a

Browse files
niksavelievniksaveliev
authored
Merge c84607a into 0ca1ac0 
2 parents 0ca1ac0 + c84607a commit 7df7f9a

File tree

2 files changed

+160
-11
lines changed

2 files changed

+160
-11
lines changed

‎ydb/public/lib/ydb_cli/commands/ydb_service_topic.cpp

+134-7
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,18 @@ namespace NYdb::NConsoleClient {
3838
std::pair<NTopic::EMeteringMode, TString>(NTopic::EMeteringMode::RequestUnits, "Read/write operations valued in request units, storage usage on hourly basis."),
3939
};
4040

41+
THashMap<TString, NTopic::EAutoscalingStrategy> AutoscaleStrategies = {
42+
std::pair<TString, NTopic::EAutoscalingStrategy>("disabled", NTopic::EAutoscalingStrategy::Disabled),
43+
std::pair<TString, NTopic::EAutoscalingStrategy>("up", NTopic::EAutoscalingStrategy::ScaleUp),
44+
std::pair<TString, NTopic::EAutoscalingStrategy>("up-and-down", NTopic::EAutoscalingStrategy::ScaleUpAndDown),
45+
};
46+
47+
THashMap<NTopic::EAutoscalingStrategy, TString> AutoscaleStrategiesDescriptions = {
48+
std::pair<NTopic::EAutoscalingStrategy, TString>(NTopic::EAutoscalingStrategy::Disabled, "Automatic scaling of the number of partitions is disabled"),
49+
std::pair<NTopic::EAutoscalingStrategy, TString>(NTopic::EAutoscalingStrategy::ScaleUp, "The number of partitions can increase under high load, but cannot decrease"),
50+
std::pair<NTopic::EAutoscalingStrategy, TString>(NTopic::EAutoscalingStrategy::ScaleUpAndDown, "The number of partitions can increase under high load and decrease under low load"),
51+
};
52+
4153
THashMap<ETopicMetadataField, TString> TopicMetadataFieldsDescriptions = {
4254
{ETopicMetadataField::Body, "Message data"},
4355
{ETopicMetadataField::WriteTime, "Message write time, a UNIX timestamp the message was written to server."},
@@ -172,6 +184,81 @@ namespace {
172184
return MeteringMode_;
173185
}
174186

187+
void TCommandWithAutoscaling::AddAutoscaling(TClientCommand::TConfig& config, bool isAlter) {
188+
TStringStream description;
189+
description << "A strategy to automatically change the number of partitions depending on the load. Available strategies: ";
190+
NColorizer::TColors colors = NColorizer::AutoColors(Cout);
191+
for (const auto& strategy: AutoscaleStrategies) {
192+
auto findResult = AutoscaleStrategiesDescriptions.find(strategy.second);
193+
Y_ABORT_UNLESS(findResult != AutoscaleStrategiesDescriptions.end(),
194+
"Couldn't find description for %s autoscale strategy", (TStringBuilder() << strategy.second).c_str());
195+
description << "\n " << colors.BoldColor() << strategy.first << colors.OldColor()
196+
<< "\n " << findResult->second;
197+
}
198+
199+
if (isAlter) {
200+
config.Opts->AddLongOption("autoscale-strategy", description.Str())
201+
.Optional()
202+
.StoreResult(&AutoscaleStrategy_);
203+
config.Opts->AddLongOption("autoscale-threshold-time", "Duration in seconds of high or low load before automatically scale the number of partitions")
204+
.Optional()
205+
.StoreResult(&ScaleThresholdTime_);
206+
config.Opts->AddLongOption("autoscale-scale-up-threshold-percent", "The load percentage at which the number of partitions will increase")
207+
.Optional()
208+
.StoreResult(&ScaleUpThresholdPercent_);
209+
config.Opts->AddLongOption("autoscale-scale-down-threshold-percent", "The load percentage at which the number of partitions will decrease")
210+
.Optional()
211+
.StoreResult(&ScaleDownThresholdPercent_);
212+
} else {
213+
config.Opts->AddLongOption("autoscale-strategy", description.Str())
214+
.Optional()
215+
.DefaultValue("disabled")
216+
.StoreResult(&AutoscaleStrategy_);
217+
config.Opts->AddLongOption("autoscale-threshold-time", "Duration in seconds of high or low load before automatically scale the number of partitions")
218+
.Optional()
219+
.DefaultValue(300)
220+
.StoreResult(&ScaleThresholdTime_);
221+
config.Opts->AddLongOption("autoscale-scale-up-threshold-percent", "The load percentage at which the number of partitions will increase")
222+
.Optional()
223+
.DefaultValue(90)
224+
.StoreResult(&ScaleUpThresholdPercent_);
225+
config.Opts->AddLongOption("autoscale-scale-down-threshold-percent", "The load percentage at which the number of partitions will decrease")
226+
.Optional()
227+
.DefaultValue(30)
228+
.StoreResult(&ScaleDownThresholdPercent_);
229+
}
230+
}
231+
232+
void TCommandWithAutoscaling::ParseAutoscalingStrategy() {
233+
if (AutoscalingStrategyStr_.empty()) {
234+
return;
235+
}
236+
237+
TString toLowerStrategy = to_lower(AutoscalingStrategyStr_);
238+
auto strategyIt = AutoscaleStrategies.find(toLowerStrategy);
239+
if (strategyIt.IsEnd()) {
240+
throw TMisuseException() << "Autoscaling strategy " << AutoscalingStrategyStr_ << " is not available for this command";
241+
} else {
242+
AutoscaleStrategy_ = strategyIt->second;
243+
}
244+
}
245+
246+
TMaybe<NTopic::EAutoscalingStrategy> TCommandWithAutoscaling::GetAutoscalingStrategy() const {
247+
return AutoscaleStrategy_;
248+
}
249+
250+
TMaybe<ui32> TCommandWithAutoscaling::GetScaleThresholdTime() const {
251+
return ScaleThresholdTime_;
252+
}
253+
254+
TMaybe<ui32> TCommandWithAutoscaling::GetScaleUpThresholdPercent() const {
255+
return ScaleUpThresholdPercent_;
256+
}
257+
258+
TMaybe<ui32> TCommandWithAutoscaling::GetScaleDownThresholdPercent() const {
259+
return ScaleDownThresholdPercent_;
260+
}
261+
175262
TCommandTopic::TCommandTopic()
176263
: TClientCommandTree("topic", {}, "TopicService operations") {
177264
AddCommand(std::make_unique<TCommandTopicCreate>());
@@ -188,9 +275,13 @@ namespace {
188275

189276
void TCommandTopicCreate::Config(TConfig& config) {
190277
TYdbCommand::Config(config);
191-
config.Opts->AddLongOption("partitions-count", "Total partitions count for topic")
278+
config.Opts->AddLongOption("partitions-count", "Initial number of partitions for topic")
192279
.DefaultValue(1)
193-
.StoreResult(&PartitionsCount_);
280+
.StoreResult(&MinActivePartitions_);
281+
config.Opts->AddLongOption("max-partitions-count", "Maximum number of partitions for topic")
282+
.DefaultValue(1)
283+
.Optional()
284+
.StoreResult(&MaxActivePartitions_);
194285
config.Opts->AddLongOption("retention-period-hours", "Duration in hours for which data in topic is stored")
195286
.DefaultValue(24)
196287
.Optional()
@@ -207,21 +298,30 @@ namespace {
207298
SetFreeArgTitle(0, "<topic-path>", "Topic path");
208299
AddAllowedCodecs(config, AllowedCodecs);
209300
AddAllowedMeteringModes(config);
301+
AddAutoscaling(config, false);
210302
}
211303

212304
void TCommandTopicCreate::Parse(TConfig& config) {
213305
TYdbCommand::Parse(config);
214306
ParseTopicName(config, 0);
215307
ParseCodecs();
216308
ParseMeteringMode();
309+
ParseAutoscalingStrategy();
217310
}
218311

219312
int TCommandTopicCreate::Run(TConfig& config) {
220313
TDriver driver = CreateDriver(config);
221314
NYdb::NTopic::TTopicClient topicClient(driver);
222315

223316
auto settings = NYdb::NTopic::TCreateTopicSettings();
224-
settings.PartitioningSettings(PartitionsCount_, PartitionsCount_);
317+
318+
auto autoscaleSettings = NTopic::TAutoscalingSettings(
319+
GetAutoscalingStrategy() ? *GetAutoscalingStrategy() : NTopic::EAutoscalingStrategy::Disabled,
320+
GetScaleThresholdTime() ? TDuration::Seconds(*GetScaleThresholdTime()) : TDuration::Seconds(0),
321+
GetScaleUpThresholdPercent() ? *GetScaleUpThresholdPercent() : 0,
322+
GetScaleDownThresholdPercent() ? *GetScaleDownThresholdPercent() : 0);
323+
324+
settings.PartitioningSettings(MinActivePartitions_, MaxActivePartitions_, autoscaleSettings);
225325
settings.PartitionWriteBurstBytes(PartitionWriteSpeedKbps_ * 1_KB);
226326
settings.PartitionWriteSpeedBytesPerSecond(PartitionWriteSpeedKbps_ * 1_KB);
227327

@@ -249,8 +349,11 @@ namespace {
249349

250350
void TCommandTopicAlter::Config(TConfig& config) {
251351
TYdbCommand::Config(config);
252-
config.Opts->AddLongOption("partitions-count", "Total partitions count for topic")
253-
.StoreResult(&PartitionsCount_);
352+
config.Opts->AddLongOption("partitions-count", "Initial number of partitions for topic")
353+
.StoreResult(&MinActivePartitions_);
354+
config.Opts->AddLongOption("max-partitions-count", "Maximum number of partitions for topic")
355+
.Optional()
356+
.StoreResult(&MaxActivePartitions_);
254357
config.Opts->AddLongOption("retention-period-hours", "Duration for which data in topic is stored")
255358
.Optional()
256359
.StoreResult(&RetentionPeriodHours_);
@@ -264,6 +367,7 @@ namespace {
264367
SetFreeArgTitle(0, "<topic-path>", "Topic path");
265368
AddAllowedCodecs(config, AllowedCodecs);
266369
AddAllowedMeteringModes(config);
370+
AddAutoscaling(config, true);
267371
}
268372

269373
void TCommandTopicAlter::Parse(TConfig& config) {
@@ -276,9 +380,32 @@ namespace {
276380
NYdb::NTopic::TAlterTopicSettings TCommandTopicAlter::PrepareAlterSettings(
277381
NYdb::NTopic::TDescribeTopicResult& describeResult) {
278382
auto settings = NYdb::NTopic::TAlterTopicSettings();
383+
auto partitioningSettings = settings.BeginAlterPartitioningSettings();
384+
385+
if (MinActivePartitions_.Defined() && (*MinActivePartitions_ != describeResult.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions())) {
386+
partitioningSettings.MinActivePartitions(*MinActivePartitions_);
387+
}
388+
389+
if (MaxActivePartitions_.Defined() && (*MaxActivePartitions_ != describeResult.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions())) {
390+
partitioningSettings.MaxActivePartitions(*MaxActivePartitions_);
391+
}
392+
393+
auto autoscalingSettings = partitioningSettings.BeginAlterAutoscalingSettings();
394+
395+
if (GetScaleThresholdTime().Defined() && *GetScaleThresholdTime() != describeResult.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetThresholdTime().Seconds()) {
396+
autoscalingSettings.ThresholdTime(TDuration::Seconds(*GetScaleThresholdTime()));
397+
}
398+
399+
if (GetAutoscalingStrategy().Defined() && *GetAutoscalingStrategy() != describeResult.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetStrategy()) {
400+
autoscalingSettings.Strategy(*GetAutoscalingStrategy());
401+
}
402+
403+
if (GetScaleDownThresholdPercent().Defined() && *GetScaleDownThresholdPercent() != describeResult.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetScaleDownThresholdPercent()) {
404+
autoscalingSettings.ScaleDownThresholdPercent(*GetScaleDownThresholdPercent());
405+
}
279406

280-
if (PartitionsCount_.Defined() && (*PartitionsCount_ != describeResult.GetTopicDescription().GetTotalPartitionsCount())) {
281-
settings.AlterPartitioningSettings(*PartitionsCount_, *PartitionsCount_);
407+
if (GetScaleUpThresholdPercent().Defined() && *GetScaleUpThresholdPercent() != describeResult.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetScaleUpThresholdPercent()) {
408+
autoscalingSettings.ScaleUpThresholdPercent(*GetScaleUpThresholdPercent());
282409
}
283410

284411
auto codecs = GetCodecs();

‎ydb/public/lib/ydb_cli/commands/ydb_service_topic.h

+26-4
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,30 @@ namespace NYdb::NConsoleClient {
3737
NTopic::EMeteringMode MeteringMode_ = NTopic::EMeteringMode::Unspecified;
3838
};
3939

40+
class TCommandWithAutoscaling {
41+
protected:
42+
void AddAutoscaling(TClientCommand::TConfig& config, bool withDefault);
43+
void ParseAutoscalingStrategy();
44+
TMaybe<NTopic::EAutoscalingStrategy> GetAutoscalingStrategy() const;
45+
TMaybe<ui32> GetScaleThresholdTime() const;
46+
TMaybe<ui32> GetScaleUpThresholdPercent() const;
47+
TMaybe<ui32> GetScaleDownThresholdPercent() const;
48+
49+
private:
50+
TMaybe<ui32> ScaleThresholdTime_;
51+
TMaybe<ui32> ScaleUpThresholdPercent_;
52+
TMaybe<ui32> ScaleDownThresholdPercent_;
53+
54+
TString AutoscalingStrategyStr_;
55+
TMaybe<NTopic::EAutoscalingStrategy> AutoscaleStrategy_;
56+
};
57+
4058
class TCommandTopic: public TClientCommandTree {
4159
public:
4260
TCommandTopic();
4361
};
4462

45-
class TCommandTopicCreate: public TYdbCommand, public TCommandWithTopicName, public TCommandWithSupportedCodecs, public TCommandWithMeteringMode {
63+
class TCommandTopicCreate: public TYdbCommand, public TCommandWithTopicName, public TCommandWithSupportedCodecs, public TCommandWithMeteringMode, public TCommandWithAutoscaling {
4664
public:
4765
TCommandTopicCreate();
4866
void Config(TConfig& config) override;
@@ -52,11 +70,13 @@ namespace NYdb::NConsoleClient {
5270
private:
5371
ui64 RetentionPeriodHours_;
5472
ui64 RetentionStorageMb_;
55-
ui32 PartitionsCount_;
73+
ui32 MinActivePartitions_;
74+
ui32 MaxActivePartitions_;
75+
5676
ui32 PartitionWriteSpeedKbps_;
5777
};
5878

59-
class TCommandTopicAlter: public TYdbCommand, public TCommandWithTopicName, public TCommandWithSupportedCodecs, public TCommandWithMeteringMode {
79+
class TCommandTopicAlter: public TYdbCommand, public TCommandWithTopicName, public TCommandWithSupportedCodecs, public TCommandWithMeteringMode, public TCommandWithAutoscaling {
6080
public:
6181
TCommandTopicAlter();
6282
void Config(TConfig& config) override;
@@ -66,7 +86,9 @@ namespace NYdb::NConsoleClient {
6686
private:
6787
TMaybe<ui64> RetentionPeriodHours_;
6888
TMaybe<ui64> RetentionStorageMb_;
69-
TMaybe<ui32> PartitionsCount_;
89+
TMaybe<ui32> MinActivePartitions_;
90+
TMaybe<ui32> MaxActivePartitions_;
91+
7092
TMaybe<ui32> PartitionWriteSpeedKbps_;
7193

7294
NYdb::NTopic::TAlterTopicSettings PrepareAlterSettings(NYdb::NTopic::TDescribeTopicResult& describeResult);

0 commit comments

Comments
 (0)