base: move TCompactionPolicy out of flat_scheme_op.proto

ydb/core/base/localdb.cpp

@@ -22,14 +22,14 @@ TCompactionPolicy::TBackgroundPolicy::TBackgroundPolicy(ui32 threshold,
     , ResourceBrokerTask(resourceBrokerTask)
 {}
 
-TCompactionPolicy::TBackgroundPolicy::TBackgroundPolicy(const NKikimrSchemeOp::TCompactionPolicy::TBackgroundPolicy &policyPb)
+TCompactionPolicy::TBackgroundPolicy::TBackgroundPolicy(const NKikimrCompaction::TCompactionPolicy::TBackgroundPolicy &policyPb)
     : Threshold(policyPb.HasThreshold() ? policyPb.GetThreshold() : 101)
     , PriorityBase(policyPb.HasPriorityBase() ? policyPb.GetPriorityBase() : 100)
     , TimeFactor(policyPb.HasTimeFactor() ? policyPb.GetTimeFactor() : 1.0)
     , ResourceBrokerTask(policyPb.HasResourceBrokerTask() ? policyPb.GetResourceBrokerTask() : BackgroundCompactionTaskName)
 {}
 
-void TCompactionPolicy::TBackgroundPolicy::Serialize(NKikimrSchemeOp::TCompactionPolicy::TBackgroundPolicy& policyPb) const
+void TCompactionPolicy::TBackgroundPolicy::Serialize(NKikimrCompaction::TCompactionPolicy::TBackgroundPolicy& policyPb) const
 {
     policyPb.SetThreshold(Threshold);
     policyPb.SetPriorityBase(PriorityBase);
@@ -59,7 +59,7 @@ TCompactionPolicy::TGenerationPolicy::TGenerationPolicy(ui64 sizeToCompact,
     , UpliftPartSize(sizeToCompact / countToCompact)
 {}
 
-TCompactionPolicy::TGenerationPolicy::TGenerationPolicy(const NKikimrSchemeOp::TCompactionPolicy::TGenerationPolicy &policyPb)
+TCompactionPolicy::TGenerationPolicy::TGenerationPolicy(const NKikimrCompaction::TCompactionPolicy::TGenerationPolicy &policyPb)
     : SizeToCompact(policyPb.HasSizeToCompact() ? policyPb.GetSizeToCompact() : 0)
     , CountToCompact(policyPb.HasCountToCompact() ? policyPb.GetCountToCompact() : 5)
     , ForceCountToCompact(policyPb.HasForceCountToCompact() ? policyPb.GetForceCountToCompact() : 8)
@@ -78,7 +78,7 @@ TCompactionPolicy::TGenerationPolicy::TGenerationPolicy(const NKikimrSchemeOp::T
         ResourceBrokerTask = LegacyQueueIdToTaskName(CompactionBrokerQueue);
 }
 
-void TCompactionPolicy::TGenerationPolicy::Serialize(NKikimrSchemeOp::TCompactionPolicy::TGenerationPolicy& policyPb) const
+void TCompactionPolicy::TGenerationPolicy::Serialize(NKikimrCompaction::TCompactionPolicy::TGenerationPolicy& policyPb) const
 {
     policyPb.SetSizeToCompact(SizeToCompact);
     policyPb.SetCountToCompact(CountToCompact);
@@ -116,11 +116,11 @@ TCompactionPolicy::TCompactionPolicy()
     , LogOverheadSizeToSnapshot(16 * 1024 * 1024)
     , LogOverheadCountToSnapshot(500)
     , DroppedRowsPercentToCompact(50)
-    , CompactionStrategy(NKikimrSchemeOp::CompactionStrategyUnset)
+    , CompactionStrategy(NKikimrCompaction::CompactionStrategyUnset)
     , KeepEraseMarkers(false)
 {}
 
-TCompactionPolicy::TCompactionPolicy(const NKikimrSchemeOp::TCompactionPolicy& policyPb)
+TCompactionPolicy::TCompactionPolicy(const NKikimrCompaction::TCompactionPolicy& policyPb)
     : InMemSizeToSnapshot(policyPb.HasInMemSizeToSnapshot() ? policyPb.GetInMemSizeToSnapshot() : 4 * 1024 * 1024)
     , InMemStepsToSnapshot(policyPb.HasInMemStepsToSnapshot() ? policyPb.GetInMemStepsToSnapshot() : 300)
     , InMemForceStepsToSnapshot(policyPb.HasInMemForceStepsToSnapshot() ? policyPb.GetInMemForceStepsToSnapshot() : 500)
@@ -161,7 +161,7 @@ TCompactionPolicy::TCompactionPolicy(const NKikimrSchemeOp::TCompactionPolicy& p
     }
 }
 
-void TCompactionPolicy::Serialize(NKikimrSchemeOp::TCompactionPolicy& policyPb) const {
+void TCompactionPolicy::Serialize(NKikimrCompaction::TCompactionPolicy& policyPb) const {
     policyPb.SetInMemSizeToSnapshot(InMemSizeToSnapshot);
     policyPb.SetInMemStepsToSnapshot(InMemStepsToSnapshot);
     policyPb.SetInMemForceStepsToSnapshot(InMemForceStepsToSnapshot);
@@ -182,7 +182,7 @@ void TCompactionPolicy::Serialize(NKikimrSchemeOp::TCompactionPolicy& policyPb)
     policyPb.SetLogOverheadSizeToSnapshot(LogOverheadSizeToSnapshot);
     policyPb.SetLogOverheadCountToSnapshot(LogOverheadCountToSnapshot);
     policyPb.SetDroppedRowsPercentToCompact(DroppedRowsPercentToCompact);
-    if (CompactionStrategy != NKikimrSchemeOp::CompactionStrategyUnset) {
+    if (CompactionStrategy != NKikimrCompaction::CompactionStrategyUnset) {
         policyPb.SetCompactionStrategy(CompactionStrategy);
     }
     if (KeepEraseMarkers) {

ydb/core/base/localdb.h

@@ -3,7 +3,7 @@
 #include <util/generic/map.h>
 #include <util/generic/hash_set.h>
 #include <util/generic/list.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
+#include <ydb/core/protos/compaction.pb.h>
 #include <ydb/library/yverify_stream/yverify_stream.h>
 #include <google/protobuf/util/message_differencer.h>
 
@@ -19,9 +19,9 @@ struct TCompactionPolicy : public TThrRefBase {
 
         TBackgroundPolicy();
         TBackgroundPolicy(ui32 threshold, ui32 priorityBase, double timeFactor, const TString &resourceBrokerTask);
-        TBackgroundPolicy(const NKikimrSchemeOp::TCompactionPolicy::TBackgroundPolicy &policyPb);
+        TBackgroundPolicy(const NKikimrCompaction::TCompactionPolicy::TBackgroundPolicy &policyPb);
 
-        void Serialize(NKikimrSchemeOp::TCompactionPolicy::TBackgroundPolicy& policyPb) const;
+        void Serialize(NKikimrCompaction::TCompactionPolicy::TBackgroundPolicy& policyPb) const;
 
         bool operator ==(const TBackgroundPolicy& p) const {
             return (Threshold == p.Threshold
@@ -50,9 +50,9 @@ struct TCompactionPolicy : public TThrRefBase {
                           ui64 forceSizeToCompact, const TString &resourceBrokerTask,
                           bool keepInCache,
                           const TBackgroundPolicy &backgroundCompactionPolicy = TBackgroundPolicy());
-        TGenerationPolicy(const NKikimrSchemeOp::TCompactionPolicy::TGenerationPolicy &policyPb);
+        TGenerationPolicy(const NKikimrCompaction::TCompactionPolicy::TGenerationPolicy &policyPb);
 
-        void Serialize(NKikimrSchemeOp::TCompactionPolicy::TGenerationPolicy& policyPb) const;
+        void Serialize(NKikimrCompaction::TCompactionPolicy::TGenerationPolicy& policyPb) const;
 
         bool operator ==(const TGenerationPolicy& p) const {
             return SizeToCompact == p.SizeToCompact
@@ -91,16 +91,16 @@ struct TCompactionPolicy : public TThrRefBase {
     ui64 LogOverheadSizeToSnapshot;
     ui32 LogOverheadCountToSnapshot;
     ui32 DroppedRowsPercentToCompact;
-    NKikimrSchemeOp::ECompactionStrategy CompactionStrategy;
-    NKikimrSchemeOp::TCompactionPolicy::TShardPolicy ShardPolicy;
+    NKikimrCompaction::ECompactionStrategy CompactionStrategy;
+    NKikimrCompaction::TCompactionPolicy::TShardPolicy ShardPolicy;
     bool KeepEraseMarkers;
 
     TVector<TGenerationPolicy> Generations;
 
     TCompactionPolicy();
-    explicit TCompactionPolicy(const NKikimrSchemeOp::TCompactionPolicy& policyPb);
+    explicit TCompactionPolicy(const NKikimrCompaction::TCompactionPolicy& policyPb);
 
-    void Serialize(NKikimrSchemeOp::TCompactionPolicy& policyPb) const;
+    void Serialize(NKikimrCompaction::TCompactionPolicy& policyPb) const;
 
     bool operator ==(const TCompactionPolicy& p) const {
         return InMemSizeToSnapshot == p.InMemSizeToSnapshot

ydb/core/base/localdb_ut.cpp

@@ -7,7 +7,7 @@ namespace NLocalDb {
     Y_UNIT_TEST_SUITE(TLocalDbTest) {
 
         Y_UNIT_TEST(BackupTaskNameChangedAtLoadTime) {
-            NKikimrSchemeOp::TCompactionPolicy policyPb;
+            NKikimrCompaction::TCompactionPolicy policyPb;
             CreateDefaultUserTablePolicy()->Serialize(policyPb);
 
             // We expect default task name to be "scan" at the moment

ydb/core/cms/console/console__scheme.h

@@ -5,6 +5,8 @@
 #include <ydb/core/scheme/scheme_types_defs.h>
 #include <ydb/core/tablet_flat/flat_cxx_database.h>
 
+#include <ydb/core/protos/flat_scheme_op.pb.h>
+
 namespace NKikimr::NConsole {
 
 struct Schema : NIceDb::Schema {

ydb/core/protos/compaction.proto

@@ -0,0 +1,113 @@
+package NKikimrCompaction;
+option java_package = "ru.yandex.kikimr.proto";
+
+enum ECompactionStrategy {
+    CompactionStrategyUnset = 0;
+    CompactionStrategyGenerational = 1;
+    CompactionStrategySharded = 2; // DEPRECATED: use CompactionStrategyGenerational
+}
+
+message TCompactionPolicy {
+
+    message TBackgroundPolicy {
+        // How much (in %) of forced compaction criteria should be met to submit background task.
+        optional uint32 Threshold = 1 [default = 101]; // no background compaction by default
+        // Base background compaction priority value (less priority means more important task).
+        // Value is used to compute real task priority basing on compaction criteria, time
+        // in queue etc.
+        optional uint32 PriorityBase = 2 [default = 100];
+        // Submitted background task may become more prioritized over time.
+        // New priority is computed as priority /= max(log(elapsed_seconds) * factor, 1);
+        optional double TimeFactor = 3 [default = 1.0];
+        optional uint32 TaskType = 4 [default = 5]; // DEPRECATED
+        optional string ResourceBrokerTask = 5;
+    }
+
+    message TGenerationPolicy {
+        optional uint32 GenerationId = 1;
+        optional uint64 SizeToCompact = 2;
+        optional uint32 CountToCompact = 3;
+        optional uint32 ForceCountToCompact = 4; // OR one of force limits happend
+        optional uint64 ForceSizeToCompact = 5;
+        optional uint32 CompactionBrokerQueue = 6; // DEPRECATED
+        optional bool KeepInCache = 7;
+        optional TBackgroundPolicy BackgroundCompactionPolicy = 8;
+        optional string ResourceBrokerTask = 9;
+        optional uint32 ExtraCompactionPercent = 10;
+        optional uint64 ExtraCompactionMinSize = 11;
+        optional uint32 ExtraCompactionExpPercent = 12;
+        optional uint64 ExtraCompactionExpMaxSize = 13;
+        optional uint64 UpliftPartSize = 14;
+    }
+
+    //NOTE: default values must be in sync with ydb/core/base/localdb.cpp, TShardPolicy
+    message TShardPolicy {
+        // Adjacent shards smaller than this will be merged
+        optional uint64 MinShardSize = 1 [default = 33554432];
+
+        // Shards bigger than this will split in two or more pieces
+        optional uint64 MaxShardSize = 2 [default = 134217728];
+
+        // Slices smaller than this will get prioritized compaction
+        optional uint64 MinSliceSize = 3 [default = 2097152];
+
+        // Level will be compacted when there are more than this number of slices
+        optional uint32 MaxSlicesPerLevel = 4 [default = 256];
+
+        // Shard will be compacted when there are more than this number of levels
+        optional uint32 MaxTotalLevels = 5 [default = 16];
+
+        // Shard will avoid compacting less than this number of levels
+        optional uint32 MinLevelsToCompact = 6 [default = 2];
+
+        // Level will be compacted when it has X% of its data in upper levels
+        optional uint32 NewDataPercentToCompact = 7 [default = 100];
+
+        // Level will be compacted when it has X% of its rows in upper levels
+        optional uint32 NewRowsPercentToCompact = 8 [default = 0];
+
+        // Resource broker task type for compactions
+        optional string ResourceBrokerTask = 9 [default = "compaction_gen1"];
+
+        // Base priority for compaction tasks
+        optional uint32 TaskPriorityBase = 10 [default = 1000];
+
+        // Task priority will be increased for every N levels over the minimum
+        optional uint32 TaskPriorityLevelsBoost = 11 [default = 1];
+
+        // Task priority will be decreased for every N bytes of input
+        optional uint64 TaskPrioritySizePenalty = 12 [default = 4194304];
+
+        // Part data may be reused, unless it would leave this much garbage
+        optional uint32 MaxGarbagePercentToReuse = 13 [default = 20];
+
+        // Minimum slice that that may be reused
+        optional uint64 MinSliceSizeToReuse = 14 [default = 524288];
+    }
+
+    optional uint64 InMemSizeToSnapshot = 1;
+    optional uint32 InMemStepsToSnapshot = 2; // snapshot inmem state when size AND steps from last snapshot passed
+    optional uint32 InMemForceStepsToSnapshot = 3; // OR steps passed
+    optional uint64 InMemForceSizeToSnapshot = 4; // OR size reached
+    optional uint32 InMemCompactionBrokerQueue = 5 [default = 0]; // DEPRECATED
+    optional uint64 ReadAheadHiThreshold = 6 [default = 67108864];
+    optional uint64 ReadAheadLoThreshold = 7 [default = 16777216];
+    optional uint32 MinDataPageSize = 8 [default = 7168];   // 7KB is smallest optimal for 512 byte sectors
+    optional uint32 SnapBrokerQueue = 9 [default = 0]; // DEPRECATED
+    optional uint32 BackupBrokerQueue = 11 [default = 1]; // DEPRECATED
+    optional uint32 DefaultTaskPriority = 12 [default = 5];
+    optional TBackgroundPolicy BackgroundSnapshotPolicy = 13;
+    optional string InMemResourceBrokerTask = 14;
+    optional string SnapshotResourceBrokerTask = 15;
+    optional string BackupResourceBrokerTask = 16;
+    optional uint64 LogOverheadSizeToSnapshot = 17;
+    optional uint32 LogOverheadCountToSnapshot = 18;
+    optional uint32 DroppedRowsPercentToCompact = 19;
+    optional ECompactionStrategy CompactionStrategy = 20 [default = CompactionStrategyUnset];
+    optional TShardPolicy ShardPolicy = 21;
+    optional bool KeepEraseMarkers = 22;
+    optional uint32 MinBTreeIndexNodeSize = 23 [default = 7168];
+    optional uint32 MinBTreeIndexNodeKeys = 24 [default = 6];
+
+    repeated TGenerationPolicy Generation = 10;
+}

ydb/core/protos/config.proto

@@ -15,6 +15,7 @@ import "ydb/core/protos/counters_schemeshard.proto";
 import "ydb/core/protos/datashard_config.proto";
 import "ydb/core/protos/drivemodel.proto";
 import "ydb/core/protos/feature_flags.proto";
+import "ydb/core/protos/compaction.proto";
 import "ydb/core/protos/flat_scheme_op.proto";
 import "ydb/core/protos/hive.proto";
 import "ydb/core/protos/http_config.proto";
@@ -230,7 +231,7 @@ message TDomainsConfig {
 
     message TNamedCompactionPolicy {
         optional string Name = 1;
-        optional NKikimrSchemeOp.TCompactionPolicy Policy = 2;
+        optional NKikimrCompaction.TCompactionPolicy Policy = 2;
     }
 
     message TSecurityConfig {
@@ -962,7 +963,7 @@ message TConfigsDispatcherConfig {
 
 message TCompactionPolicy {
     optional string Name = 1;
-    optional NKikimrSchemeOp.TCompactionPolicy CompactionPolicy = 2;
+    optional NKikimrCompaction.TCompactionPolicy CompactionPolicy = 2;
 }
 
 message TExecutionPolicy {