Skip to content

Commit 0e2c184

Browse files
niksavelievniksaveliev
authored
Merge df3fd33 into 9910c50 
2 parents 9910c50 + df3fd33 commit 0e2c184

File tree

20 files changed

+472
-254
lines changed

20 files changed

+472
-254
lines changed

‎ydb/core/persqueue/ut/common/autoscaling_ut_common.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,15 +124,15 @@ std::shared_ptr<ISimpleBlockingWriteSession> CreateWriteSession(TTopicClient& cl
124124
}
125125

126126

127-
TTestReadSession::TTestReadSession(const TString& name, TTopicClient& client, size_t expectedMessagesCount, bool autoCommit, std::set<ui32> partitions, bool autoscalingSupport) {
127+
TTestReadSession::TTestReadSession(const TString& name, TTopicClient& client, size_t expectedMessagesCount, bool autoCommit, std::set<ui32> partitions, bool autoPartitioningSupport) {
128128
Impl = std::make_shared<TImpl>(name, autoCommit);
129129

130130
Impl->Acquire();
131131

132132
auto readSettings = TReadSessionSettings()
133133
.ConsumerName(TEST_CONSUMER)
134134
.AppendTopics(TEST_TOPIC)
135-
.AutoscalingSupport(autoscalingSupport);
135+
.AutoPartitioningSupport(autoPartitioningSupport);
136136
for (auto partitionId : partitions) {
137137
readSettings.Topics_[0].AppendPartitionIds(partitionId);
138138
}

‎ydb/core/persqueue/ut/common/autoscaling_ut_common.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ struct TTestReadSession {
4242

4343
static constexpr size_t SemCount = 1;
4444

45-
TTestReadSession(const TString& name, TTopicClient& client, size_t expectedMessagesCount = Max<size_t>(), bool autoCommit = true, std::set<ui32> partitions = {}, bool autoscalingSupport = true);
45+
TTestReadSession(const TString& name, TTopicClient& client, size_t expectedMessagesCount = Max<size_t>(), bool autoCommit = true, std::set<ui32> partitions = {}, bool autoPartitioningSupport = true);
4646

4747
void WaitAllMessages();
4848

‎ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -631,19 +631,19 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) {
631631
auto scaleUpPercent = 80;
632632
auto scaleDownPercent = 20;
633633
auto threshold = 500;
634-
auto strategy = EAutoscalingStrategy::ScaleUp;
634+
auto strategy = EAutoPartitioningStrategy::ScaleUp;
635635

636636
TCreateTopicSettings createSettings;
637637
createSettings
638638
.BeginConfigurePartitioningSettings()
639639
.MinActivePartitions(minParts)
640640
.MaxActivePartitions(maxParts)
641-
.BeginConfigureAutoscalingSettings()
642-
.ScaleUpThresholdPercent(scaleUpPercent)
643-
.ScaleDownThresholdPercent(scaleDownPercent)
644-
.ThresholdTime(TDuration::Seconds(threshold))
641+
.BeginConfigureAutoPartitioningSettings()
642+
.UpUtilizationPercent(scaleUpPercent)
643+
.DownUtilizationPercent(scaleDownPercent)
644+
.StabilizationWindow(TDuration::Seconds(threshold))
645645
.Strategy(strategy)
646-
.EndConfigureAutoscalingSettings()
646+
.EndConfigureAutoPartitioningSettings()
647647
.EndConfigurePartitioningSettings();
648648
client.CreateTopic(autoscalingTestTopic, createSettings).Wait();
649649

@@ -655,29 +655,29 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) {
655655

656656
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), minParts);
657657
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), maxParts);
658-
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetStrategy(), strategy);
659-
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetScaleDownThresholdPercent(), scaleDownPercent);
660-
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetScaleUpThresholdPercent(), scaleUpPercent);
661-
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetThresholdTime().Seconds(), threshold);
658+
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), strategy);
659+
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), scaleDownPercent);
660+
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), scaleUpPercent);
661+
UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), threshold);
662662

663663
auto alterMinParts = 10;
664664
auto alterMaxParts = 20;
665665
auto alterScaleUpPercent = 90;
666666
auto alterScaleDownPercent = 10;
667667
auto alterThreshold = 700;
668-
auto alterStrategy = EAutoscalingStrategy::ScaleUpAndDown;
668+
auto alterStrategy = EAutoPartitioningStrategy::ScaleUpAndDown;
669669

670670
TAlterTopicSettings alterSettings;
671671
alterSettings
672672
.BeginAlterPartitioningSettings()
673673
.MinActivePartitions(alterMinParts)
674674
.MaxActivePartitions(alterMaxParts)
675-
.BeginAlterAutoscalingSettings()
676-
.ScaleDownThresholdPercent(alterScaleDownPercent)
677-
.ScaleUpThresholdPercent(alterScaleUpPercent)
678-
.ThresholdTime(TDuration::Seconds(alterThreshold))
675+
.BeginAlterAutoPartitioningSettings()
676+
.DownUtilizationPercent(alterScaleDownPercent)
677+
.UpUtilizationPercent(alterScaleUpPercent)
678+
.StabilizationWindow(TDuration::Seconds(alterThreshold))
679679
.Strategy(alterStrategy)
680-
.EndAlterAutoscalingSettings()
680+
.EndAlterAutoPartitioningSettings()
681681
.EndAlterTopicPartitioningSettings();
682682

683683
client.AlterTopic(autoscalingTestTopic, alterSettings).Wait();
@@ -686,10 +686,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) {
686686

687687
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), alterMinParts);
688688
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), alterMaxParts);
689-
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetStrategy(), alterStrategy);
690-
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetScaleDownThresholdPercent(), alterScaleDownPercent);
691-
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetScaleUpThresholdPercent(), alterScaleUpPercent);
692-
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoscalingSettings().GetThresholdTime().Seconds(), alterThreshold);
689+
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), alterStrategy);
690+
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), alterScaleUpPercent);
691+
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), alterScaleDownPercent);
692+
UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), alterThreshold);
693693
}
694694

695695
Y_UNIT_TEST(ControlPlane_DisableAutoPartitioning) {
@@ -704,9 +704,9 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) {
704704
.BeginConfigurePartitioningSettings()
705705
.MinActivePartitions(1)
706706
.MaxActivePartitions(100)
707-
.BeginConfigureAutoscalingSettings()
708-
.Strategy(EAutoscalingStrategy::ScaleUp)
709-
.EndConfigureAutoscalingSettings()
707+
.BeginConfigureAutoPartitioningSettings()
708+
.Strategy(EAutoPartitioningStrategy::ScaleUp)
709+
.EndConfigureAutoPartitioningSettings()
710710
.EndConfigurePartitioningSettings();
711711
client.CreateTopic(topicName, createSettings).Wait();
712712
}
@@ -715,9 +715,9 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) {
715715
TAlterTopicSettings alterSettings;
716716
alterSettings
717717
.BeginAlterPartitioningSettings()
718-
.BeginAlterAutoscalingSettings()
719-
.Strategy(EAutoscalingStrategy::Disabled)
720-
.EndAlterAutoscalingSettings()
718+
.BeginAlterAutoPartitioningSettings()
719+
.Strategy(EAutoPartitioningStrategy::Disabled)
720+
.EndAlterAutoPartitioningSettings()
721721
.EndAlterTopicPartitioningSettings();
722722
auto f = client.AlterTopic(topicName, alterSettings);
723723
f.Wait();
@@ -731,9 +731,9 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) {
731731
alterSettings
732732
.BeginAlterPartitioningSettings()
733733
.MaxActivePartitions(0)
734-
.BeginAlterAutoscalingSettings()
735-
.Strategy(EAutoscalingStrategy::Disabled)
736-
.EndAlterAutoscalingSettings()
734+
.BeginAlterAutoPartitioningSettings()
735+
.Strategy(EAutoPartitioningStrategy::Disabled)
736+
.EndAlterAutoPartitioningSettings()
737737
.EndAlterTopicPartitioningSettings();
738738
auto f = client.AlterTopic(topicName, alterSettings);
739739
f.Wait();
@@ -752,9 +752,9 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) {
752752
createSettings
753753
.RetentionStorageMb(1024)
754754
.BeginConfigurePartitioningSettings()
755-
.BeginConfigureAutoscalingSettings()
756-
.Strategy(EAutoscalingStrategy::ScaleUp)
757-
.EndConfigureAutoscalingSettings()
755+
.BeginConfigureAutoPartitioningSettings()
756+
.Strategy(EAutoPartitioningStrategy::ScaleUp)
757+
.EndConfigureAutoPartitioningSettings()
758758
.EndConfigurePartitioningSettings();
759759
auto result = client.CreateTopic(autoscalingTestTopic, createSettings).GetValueSync();
760760

@@ -782,12 +782,12 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) {
782782
.BeginConfigurePartitioningSettings()
783783
.MinActivePartitions(1)
784784
.MaxActivePartitions(100)
785-
.BeginConfigureAutoscalingSettings()
786-
.ScaleUpThresholdPercent(2)
787-
.ScaleDownThresholdPercent(1)
788-
.ThresholdTime(TDuration::Seconds(1))
789-
.Strategy(EAutoscalingStrategy::ScaleUp)
790-
.EndConfigureAutoscalingSettings()
785+
.BeginConfigureAutoPartitioningSettings()
786+
.UpUtilizationPercent(2)
787+
.DownUtilizationPercent(1)
788+
.StabilizationWindow(TDuration::Seconds(1))
789+
.Strategy(EAutoPartitioningStrategy::ScaleUp)
790+
.EndConfigureAutoPartitioningSettings()
791791
.EndConfigurePartitioningSettings();
792792
client.CreateTopic(TEST_TOPIC, createSettings).Wait();
793793

‎ydb/public/api/protos/ydb_persqueue_v1.proto

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1082,16 +1082,16 @@ message Credentials {
10821082
}
10831083
}
10841084

1085-
enum AutoscalingStrategy {
1086-
// The autoscaling algorithm is not specified. The default value will be used.
1087-
AUTOSCALING_STRATEGY_UNSPECIFIED = 0;
1088-
// The autoscaling is disabled.
1089-
AUTOSCALING_STRATEGY_DISABLED = 1;
1090-
// The autoscaling algorithm will increase partitions count depending on the load characteristics.
1091-
// The autoscaling algorithm will never decrease the number of partitions.
1092-
AUTOSCALING_STRATEGY_SCALE_UP = 2;
1093-
// The autoscaling algorithm will both increase and decrease partitions count depending on the load characteristics.
1094-
AUTOSCALING_STRATEGY_SCALE_UP_AND_DOWN = 3;
1085+
enum AutoPartitioningStrategy {
1086+
// The auto partitioning algorithm is not specified. The default value will be used.
1087+
AUTO_PARTITIONING_STRATEGY_UNSPECIFIED = 0;
1088+
// The auto partitioning is disabled.
1089+
AUTO_PARTITIONING_STRATEGY_DISABLED = 1;
1090+
// The auto partitioning algorithm will increase partitions count depending on the load characteristics.
1091+
// The auto partitioning algorithm will never decrease the number of partitions.
1092+
AUTO_PARTITIONING_STRATEGY_SCALE_UP = 2;
1093+
// The auto partitioning algorithm will both increase and decrease partitions count depending on the load characteristics.
1094+
AUTO_PARTITIONING_STRATEGY_SCALE_UP_AND_DOWN = 3;
10951095
}
10961096

10971097
/**
@@ -1103,12 +1103,12 @@ message TopicSettings {
11031103
FORMAT_BASE = 1;
11041104
}
11051105

1106-
oneof partitioning {
1107-
// How many partitions in topic. Must less than database limit. Default limit - 10.
1108-
int32 partitions_count = 1 [(value) = "> 0"];
1109-
// Settings for the partitions count autoscaling.
1110-
AutoscalingSettings autoscaling_settings = 15;
1111-
};
1106+
1107+
// How many partitions in topic. Must less than database limit. Default limit - 10.
1108+
int32 partitions_count = 1 [(value) = "> 0"];
1109+
// Settings for the partitions count auto partitioning.
1110+
AutoPartitioningSettings auto_partitioning_settings = 15;
1111+
11121112

11131113
oneof retention {
11141114
// How long data in partition should be stored. Must be greater than 0 and less than limit for this database.
@@ -1184,9 +1184,9 @@ message TopicSettings {
11841184
RemoteMirrorRule remote_mirror_rule = 11;
11851185
}
11861186

1187-
message AutoscalingSettings {
1188-
// Strategy of autoscaling.
1189-
AutoscalingStrategy strategy = 1;
1187+
message AutoPartitioningSettings {
1188+
// Strategy of auto partitioning.
1189+
AutoPartitioningStrategy strategy = 1;
11901190

11911191
// Auto merge would stop working when the partitions count reaches min_active_partitions.
11921192
// Zero value means default - 1.
@@ -1198,24 +1198,24 @@ message AutoscalingSettings {
11981198
// Zero value means default - 100.
11991199
int64 partition_count_limit = 4 [(Ydb.value) = ">= 0", deprecated = true];
12001200

1201-
// Partition write speed autoscaling options.
1202-
AutoscalingPartitionWriteSpeedStrategy partition_write_speed = 5;
1201+
// Partition write speed auto partitioning options.
1202+
AutoPartitioningWriteSpeedStrategy partition_write_speed = 5;
12031203
}
12041204

1205-
message AutoscalingPartitionWriteSpeedStrategy {
1206-
//Partition will be autoscaled up (divided into 2 partitions)
1207-
//after write speed to the partition exceeds scale_up_threshold_percent (in percentage of maximum write speed to the partition) for the period of time threshold_time_seconds
1205+
message AutoPartitioningWriteSpeedStrategy {
1206+
//Partition will be auto partitioning up (divided into 2 partitions)
1207+
//after write speed to the partition exceeds up_utilization_percent (in percentage of maximum write speed to the partition) for the period of time stabilization_window
12081208

1209-
//Partition will become a candidate to the autoscaling down
1210-
//after write speed doesn’t reach scale_down_threshold_percent (in percentage of maximum write speed to the partition) for the period of time threshold_time_seconds
1211-
//This candidate partition will be autoscaled down when other neighbour partition will become a candidate to the autoscaling down and not earlier than a retention period.
1209+
//Partition will become a candidate to the auto partitioning down
1210+
//after write speed doesn’t reach down_utilization_percent (in percentage of maximum write speed to the partition) for the period of time stabilization_window
1211+
//This candidate partition will be autoscaled down when other neighbour partition will become a candidate to the auto partitioning down and not earlier than a retention period.
12121212

12131213
// Zero value means default - 300.
1214-
google.protobuf.Duration threshold_time = 1;
1214+
google.protobuf.Duration stabilization_window = 1;
12151215
// Zero value means default - 90.
1216-
int32 scale_up_threshold_percent = 2 [(Ydb.value) = ">= 0"];
1216+
int32 up_utilization_percent = 2 [(Ydb.value) = ">= 0"];
12171217
// Zero value means default - 30.
1218-
int32 scale_down_threshold_percent = 3 [(Ydb.value) = ">= 0"];
1218+
int32 down_utilization_percent = 3 [(Ydb.value) = ">= 0"];
12191219
}
12201220

12211221
/**

0 commit comments

Comments
 (0)