Synced SDK from ydb repo

Author: Gazizonoki

.gitignore

@@ -9,6 +9,7 @@ install_manifest.txt
 compile_commands.json
 CTestTestfile.cmake
 _deps
+.clangd
 
 # Ignore generated binaries -- any file without extension
 # Ignore all

src/api/grpc/draft/ydb_ymq_v1.proto

@@ -10,14 +10,19 @@ option java_package = "com.yandex.ydb.ymq.v1";
 
 
 service YmqService {
-    rpc GetQueueUrl(GetQueueUrlRequest) returns (GetQueueUrlResponse);
-    rpc CreateQueue(CreateQueueRequest) returns (CreateQueueResponse);
-    rpc SendMessage(SendMessageRequest) returns (SendMessageResponse);
-    rpc ReceiveMessage(ReceiveMessageRequest) returns (ReceiveMessageResponse);
-    rpc GetQueueAttributes(GetQueueAttributesRequest) returns (GetQueueAttributesResponse);
-    rpc ListQueues(ListQueuesRequest) returns (ListQueuesResponse);
-    rpc DeleteMessage(DeleteMessageRequest) returns (DeleteMessageResponse);
-    rpc PurgeQueue(PurgeQueueRequest) returns (PurgeQueueResponse);
-    rpc DeleteQueue(DeleteQueueRequest) returns (DeleteQueueResponse);
-    rpc ChangeMessageVisibility(ChangeMessageVisibilityRequest) returns (ChangeMessageVisibilityResponse);
+    rpc YmqGetQueueUrl(GetQueueUrlRequest) returns (GetQueueUrlResponse);
+    rpc YmqCreateQueue(CreateQueueRequest) returns (CreateQueueResponse);
+    rpc YmqSendMessage(SendMessageRequest) returns (SendMessageResponse);
+    rpc YmqReceiveMessage(ReceiveMessageRequest) returns (ReceiveMessageResponse);
+    rpc YmqGetQueueAttributes(GetQueueAttributesRequest) returns (GetQueueAttributesResponse);
+    rpc YmqListQueues(ListQueuesRequest) returns (ListQueuesResponse);
+    rpc YmqDeleteMessage(DeleteMessageRequest) returns (DeleteMessageResponse);
+    rpc YmqPurgeQueue(PurgeQueueRequest) returns (PurgeQueueResponse);
+    rpc YmqDeleteQueue(DeleteQueueRequest) returns (DeleteQueueResponse);
+    rpc YmqChangeMessageVisibility(ChangeMessageVisibilityRequest) returns (ChangeMessageVisibilityResponse);
+    rpc YmqSetQueueAttributes(SetQueueAttributesRequest) returns (SetQueueAttributesResponse);
+    rpc YmqSendMessageBatch(SendMessageBatchRequest) returns (SendMessageBatchResponse);
+    rpc YmqDeleteMessageBatch(DeleteMessageBatchRequest) returns (DeleteMessageBatchResponse);
+    rpc YmqChangeMessageVisibilityBatch(ChangeMessageVisibilityBatchRequest) returns (ChangeMessageVisibilityBatchResponse);
+    rpc YmqListDeadLetterSourceQueues(ListDeadLetterSourceQueuesRequest) returns (ListDeadLetterSourceQueuesResponse);
 }

src/api/grpc/ydb_cms_v1.proto

@@ -27,4 +27,7 @@ service CmsService {
 
     // Describe supported database options.
     rpc DescribeDatabaseOptions(Cms.DescribeDatabaseOptionsRequest) returns (Cms.DescribeDatabaseOptionsResponse);
+
+    // Get resources scale recommendation for database.
+    rpc GetScaleRecommendation(Cms.GetScaleRecommendationRequest) returns (Cms.GetScaleRecommendationResponse);
 }

src/api/protos/draft/fq.proto

@@ -442,6 +442,7 @@ message DataStreams {
     string endpoint = 3 [(Ydb.length).le = 1024];
     string database = 4 [(Ydb.length).le = 1024];
     bool secure = 5;
+    bool shared_reading = 6;
 }
 
 message Monitoring {

src/api/protos/draft/ymq.proto

@@ -25,7 +25,7 @@ message ChangeMessageVisibilityResult {
 message ChangeMessageVisibilityBatchRequestEntry {
     string id = 1;
     string receipt_handle = 2;
-    int32 visibility_timeout = 3;
+    optional int32 visibility_timeout = 3;
 }
 
 message ChangeMessageVisibilityBatchRequest {
@@ -128,7 +128,7 @@ message GetQueueAttributesResult {
 message GetQueueUrlRequest {
     Ydb.Operations.OperationParams operation_params = 1;
     string queue_name = 2;
-    string queue_owner_aws_account_id = 3;
+    optional string queue_owner_aws_account_id = 3;
 }
 
 message GetQueueUrlResponse {
@@ -141,9 +141,9 @@ message GetQueueUrlResult {
 
 message ListQueuesRequest {
     Ydb.Operations.OperationParams operation_params = 1;
-    int64 max_results = 2;
-    string next_token = 3;
-    string queue_name_prefix = 4;
+    optional int64 max_results = 2;
+    optional string next_token = 3;
+    optional string queue_name_prefix = 4;
 }
 
 message ListQueuesResponse {
@@ -178,13 +178,13 @@ message MessageAttribute {
 message ReceiveMessageRequest {
     Ydb.Operations.OperationParams operation_params = 1;
     repeated string attribute_names = 2;
-    int32 max_number_of_messages = 3;
+    optional int32 max_number_of_messages = 3;
     repeated string message_attribute_names = 4;
     repeated string message_system_attribute_names = 5;
     string queue_url = 6;
-    string receive_request_attempt_id = 7;
-    int32 visibility_timeout = 8;
-    int32 wait_time_seconds = 9;
+    optional string receive_request_attempt_id = 7;
+    optional int32 visibility_timeout = 8;
+    optional int32 wait_time_seconds = 9;
 }
 
 message ReceiveMessageResponse {
@@ -207,11 +207,11 @@ message ReceiveMessageResult {
 
 message SendMessageRequest {
     Ydb.Operations.OperationParams operation_params = 1;
-    int32 delay_seconds = 2;
+    optional int32 delay_seconds = 2;
     map<string, MessageAttribute> message_attributes = 3;
     string message_body = 4;
-    string message_deduplication_id = 5;
-    string message_group_id = 6;
+    optional string message_deduplication_id = 5;
+    optional string message_group_id = 6;
     map<string, MessageAttribute> message_system_attributes = 7;
     string queue_url = 8;
 }
@@ -228,31 +228,73 @@ message SendMessageResult {
     string sequence_number = 5;
 }
 
-message SendMessageBatchRequest {
-    Ydb.Operations.OperationParams operation_params = 1;
-    repeated SendMessageRequest entries = 2;
-}
-
-message SendMessageBatchResponse {
-    Ydb.Operations.Operation operation = 1;
-}
-
 message BatchResultErrorEntry {
     string code = 1;
     string id = 2;
     bool sender_fault = 3;
     string message = 4;
 }
 
+message SendMessageBatchRequestEntry {
+    string id = 1;
+    optional int32 delay_seconds = 2;
+    map<string, MessageAttribute> message_attributes = 3;
+    string message_body = 4;
+    optional string message_deduplication_id = 5;
+    optional string message_group_id = 6;
+    map<string, MessageAttribute> message_system_attributes = 7;
+    string queue_url = 8;
+}
+
 message SendMessageBatchResultEntry {
-    string md5_of_message_attributes = 1;
-    string md5_of_message_body= 2;
-    string md5_of_message_system_attributes= 3;
-    string message_id = 4;
-    string sequence_number = 5;
+    string id = 1;
+    string md5_of_message_body = 2;
+    string message_id = 3;
+    string md5_of_message_attributes = 4;
+    string md5_of_message_system_attributes = 5;
+    string sequence_number = 6;
+}
+
+message SendMessageBatchRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+    repeated SendMessageBatchRequestEntry entries = 2;
+    string queue_url = 3;
+}
+
+message SendMessageBatchResponse {
+    Ydb.Operations.Operation operation = 1;
 }
 
 message SendMessageBatchResult {
     repeated BatchResultErrorEntry failed = 1;
     repeated SendMessageBatchResultEntry successful = 2;
 }
+
+message SetQueueAttributesRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+    map<string, string> attributes = 2;
+    string queue_url = 3;
+}
+
+message SetQueueAttributesResponse {
+    Ydb.Operations.Operation operation = 1;
+}
+
+message SetQueueAttributesResult {
+}
+
+message ListDeadLetterSourceQueuesRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+    optional int32 max_results = 2;
+    optional string next_token = 3;
+    string queue_url = 4;
+}
+
+message ListDeadLetterSourceQueuesResponse {
+    Ydb.Operations.Operation operation = 1;
+}
+
+message ListDeadLetterSourceQueuesResult {
+    string next_token = 1;
+    repeated string queue_urls = 2;
+}

src/api/protos/ydb_cms.proto

@@ -4,7 +4,9 @@ option cc_enable_arenas = true;
 package Ydb.Cms;
 option java_package = "com.yandex.ydb.cms";
 
+import "src/api/protos/ydb_issue_message.proto";
 import "src/api/protos/ydb_operation.proto";
+import "src/api/protos/ydb_status_codes.proto";
 
 // A set of uniform storage units.
 // Single storage unit can be thought of as a reserved part of a RAID.
@@ -104,6 +106,23 @@ message DatabaseQuotas {
     repeated StorageQuotas storage_quotas = 6;
 }
 
+// A policy that is used for resource scale recommendation. If multiple are used,
+// recommender combines them to recommend the largest scale.
+message ScaleRecommenderPolicy {
+    // Policy that tracks metric and reactively recommend to adjust resources scale
+    // to keep metric close to the specified target value.
+    message TargetTrackingPolicy {
+        oneof target {
+            // A percentage of compute resources' average CPU utilization.
+            uint32 average_cpu_utilization_percent = 1; 
+        }
+    }
+
+    oneof policy {
+        TargetTrackingPolicy target_tracking_policy = 1;
+    }
+}
+
 // Request to create a new database. For successfull creation
 // specified database shouldn't exist. At least one storage
 // unit should be requested for the database.
@@ -129,6 +148,8 @@ message CreateDatabaseRequest {
     string idempotency_key = 9;
     // Optional quotas for the database
     DatabaseQuotas database_quotas = 10;
+    // Optional scale recommender policies
+    repeated ScaleRecommenderPolicy scale_recommender_policies = 11;
 }
 
 message CreateDatabaseResponse {
@@ -179,6 +200,8 @@ message GetDatabaseStatusResult {
     SchemaOperationQuotas schema_operation_quotas = 9;
     // Current quotas for the database
     DatabaseQuotas database_quotas = 10;
+    // Current scale recommender policies
+    repeated ScaleRecommenderPolicy scale_recommender_policies = 11;
 }
 
 // Change resources allocated for database.
@@ -207,6 +230,8 @@ message AlterDatabaseRequest {
     DatabaseQuotas database_quotas = 11;
     // Alter attributes. Leave the value blank to drop an attribute.
     map<string, string> alter_attributes = 12;
+    // Alter scale recommender policies.
+    repeated ScaleRecommenderPolicy scale_recommender_policies = 13;
 }
 
 message AlterDatabaseResponse {
@@ -271,3 +296,16 @@ message DescribeDatabaseOptionsResult {
     repeated AvailabilityZoneDescription availability_zones = 2;
     repeated ComputationalUnitDescription computational_units = 3;
 }
+
+// Get resources scale recommendation for database.
+message GetScaleRecommendationRequest {
+    // Required. Full path to database's home dir.
+    string path = 1;
+}
+
+message GetScaleRecommendationResponse {
+    StatusIds.StatusCode status = 1;
+    repeated Ydb.Issue.IssueMessage issues = 2;
+    // Recommended resources scale to be allocated for database.
+    Resources recommended_resources = 3;
+}

src/client/federated_topic/ut/fds_mock.h

@@ -2,7 +2,7 @@
 
 #include "util/string/builder.h"
 #include <library/cpp/threading/future/future.h>
-#include <ydb/public/api/grpc/ydb_federation_discovery_v1.grpc.pb.h>
+#include <src/api/grpc/ydb_federation_discovery_v1.grpc.pb.h>
 
 #include <deque>
 #include <optional>

src/client/federated_topic/ut/ya.make

@@ -137,7 +137,7 @@ void TSchemeEntry::Out(IOutputStream& out) const {
 void TSchemeEntry::SerializeTo(::Ydb::Scheme::ModifyPermissionsRequest& request) const {
     request.mutable_actions()->Add()->set_change_owner(TStringType{Owner});
     for (const auto& permission : Permissions) {
-    permission.SerializeTo(*request.mutable_actions()->Add()->mutable_grant());
+        permission.SerializeTo(*request.mutable_actions()->Add()->mutable_grant());
     }
 }
 

src/client/scheme/scheme.cpp

@@ -43,6 +43,20 @@ void UuidToString(ui16 dw[8], IOutputStream& out) {
     WriteHex(dw[7], out, true);
 }
 
+std::string UuidBytesToString(const std::string& in) {
+    TStringStream ss;
+    
+    UuidBytesToString(TString(in), ss);
+
+    return std::string(ss.Str());
+}
+
+void UuidBytesToString(const std::string& in, IOutputStream& out) {
+    ui16 dw[8];
+    std::memcpy(dw, in.data(), sizeof(dw));
+    NUuid::UuidToString(dw, out);
+}
+
 void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out) {
     union {
         char bytes[16];

src/library/uuid/uuid.cpp

@@ -13,6 +13,8 @@ namespace NUuid {
 
 static constexpr ui32 UUID_LEN = 16;
 
+std::string UuidBytesToString(const std::string& in);
+void UuidBytesToString(const std::string& in, IOutputStream& out);
 void UuidToString(ui16 dw[8], IOutputStream& out);
 void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out);