Sync sdks 7

.gitignore

@@ -9,6 +9,7 @@ install_manifest.txt
 compile_commands.json
 CTestTestfile.cmake
 _deps
+.clangd
 
 # Ignore generated binaries -- any file without extension
 # Ignore all

include/ydb-cpp-sdk/client/import/import.h

@@ -41,6 +41,7 @@ struct TImportFromS3Settings : public TOperationRequestSettings<TImportFromS3Set
     FLUENT_SETTING_VECTOR(TItem, Item);
     FLUENT_SETTING_OPTIONAL(std::string, Description);
     FLUENT_SETTING_OPTIONAL(uint32_t, NumberOfRetries);
+    FLUENT_SETTING_OPTIONAL(bool, NoACL);
 };
 
 class TImportFromS3Response : public TOperation {

include/ydb-cpp-sdk/client/scheme/scheme.h

@@ -24,6 +24,8 @@ struct TPermissions {
         : Subject(subject)
         , PermissionNames(names)
     {}
+    TPermissions(const ::Ydb::Scheme::Permissions& proto);
+
     std::string Subject;
     std::vector<std::string> PermissionNames;
 
@@ -139,6 +141,9 @@ struct TModifyPermissionsSettings : public TOperationRequestSettings<TModifyPerm
     void AddAction(EModifyPermissionsAction action, const TPermissions& permissions) {
         Actions_.emplace_back(std::pair<EModifyPermissionsAction, TPermissions>{action, permissions});
     }
+
+    TModifyPermissionsSettings() = default;
+    explicit TModifyPermissionsSettings(const ::Ydb::Scheme::ModifyPermissionsRequest& request);
 };
 
 class TSchemeClient {

include/ydb-cpp-sdk/library/yql_common/issue/yql_issue.h

@@ -58,13 +58,18 @@ struct TPosition {
 
 class TTextWalker {
 public:
-    TTextWalker(TPosition& position)
+    TTextWalker(TPosition& position, bool utf8Aware)
         : Position(position)
+        , Utf8Aware(utf8Aware)
         , HaveCr(false)
         , LfCount(0)
     {
     }
 
+    static inline bool IsUtf8Intermediate(char c) {
+        return (c & 0xC0) == 0x80;
+    }
+
     template<typename T>
     TTextWalker& Advance(const T& buf) {
         for (char c : buf) {
@@ -77,6 +82,7 @@ class TTextWalker {
 
 private:
     TPosition& Position;
+    const bool Utf8Aware;
     bool HaveCr;
     uint32_t LfCount;
 };

src/api/grpc/draft/ydb_ymq_v1.proto

@@ -10,14 +10,19 @@ option java_package = "com.yandex.ydb.ymq.v1";
 
 
 service YmqService {
-    rpc GetQueueUrl(GetQueueUrlRequest) returns (GetQueueUrlResponse);
-    rpc CreateQueue(CreateQueueRequest) returns (CreateQueueResponse);
-    rpc SendMessage(SendMessageRequest) returns (SendMessageResponse);
-    rpc ReceiveMessage(ReceiveMessageRequest) returns (ReceiveMessageResponse);
-    rpc GetQueueAttributes(GetQueueAttributesRequest) returns (GetQueueAttributesResponse);
-    rpc ListQueues(ListQueuesRequest) returns (ListQueuesResponse);
-    rpc DeleteMessage(DeleteMessageRequest) returns (DeleteMessageResponse);
-    rpc PurgeQueue(PurgeQueueRequest) returns (PurgeQueueResponse);
-    rpc DeleteQueue(DeleteQueueRequest) returns (DeleteQueueResponse);
-    rpc ChangeMessageVisibility(ChangeMessageVisibilityRequest) returns (ChangeMessageVisibilityResponse);
+    rpc YmqGetQueueUrl(GetQueueUrlRequest) returns (GetQueueUrlResponse);
+    rpc YmqCreateQueue(CreateQueueRequest) returns (CreateQueueResponse);
+    rpc YmqSendMessage(SendMessageRequest) returns (SendMessageResponse);
+    rpc YmqReceiveMessage(ReceiveMessageRequest) returns (ReceiveMessageResponse);
+    rpc YmqGetQueueAttributes(GetQueueAttributesRequest) returns (GetQueueAttributesResponse);
+    rpc YmqListQueues(ListQueuesRequest) returns (ListQueuesResponse);
+    rpc YmqDeleteMessage(DeleteMessageRequest) returns (DeleteMessageResponse);
+    rpc YmqPurgeQueue(PurgeQueueRequest) returns (PurgeQueueResponse);
+    rpc YmqDeleteQueue(DeleteQueueRequest) returns (DeleteQueueResponse);
+    rpc YmqChangeMessageVisibility(ChangeMessageVisibilityRequest) returns (ChangeMessageVisibilityResponse);
+    rpc YmqSetQueueAttributes(SetQueueAttributesRequest) returns (SetQueueAttributesResponse);
+    rpc YmqSendMessageBatch(SendMessageBatchRequest) returns (SendMessageBatchResponse);
+    rpc YmqDeleteMessageBatch(DeleteMessageBatchRequest) returns (DeleteMessageBatchResponse);
+    rpc YmqChangeMessageVisibilityBatch(ChangeMessageVisibilityBatchRequest) returns (ChangeMessageVisibilityBatchResponse);
+    rpc YmqListDeadLetterSourceQueues(ListDeadLetterSourceQueuesRequest) returns (ListDeadLetterSourceQueuesResponse);
 }

src/api/grpc/ydb_cms_v1.proto

@@ -27,4 +27,7 @@ service CmsService {
 
     // Describe supported database options.
     rpc DescribeDatabaseOptions(Cms.DescribeDatabaseOptionsRequest) returns (Cms.DescribeDatabaseOptionsResponse);
+
+    // Get resources scale recommendation for database.
+    rpc GetScaleRecommendation(Cms.GetScaleRecommendationRequest) returns (Cms.GetScaleRecommendationResponse);
 }

src/api/protos/draft/fq.proto

@@ -442,6 +442,7 @@ message DataStreams {
     string endpoint = 3 [(Ydb.length).le = 1024];
     string database = 4 [(Ydb.length).le = 1024];
     bool secure = 5;
+    bool shared_reading = 6;
 }
 
 message Monitoring {

src/api/protos/draft/ymq.proto

@@ -25,7 +25,7 @@ message ChangeMessageVisibilityResult {
 message ChangeMessageVisibilityBatchRequestEntry {
     string id = 1;
     string receipt_handle = 2;
-    int32 visibility_timeout = 3;
+    optional int32 visibility_timeout = 3;
 }
 
 message ChangeMessageVisibilityBatchRequest {
@@ -128,7 +128,7 @@ message GetQueueAttributesResult {
 message GetQueueUrlRequest {
     Ydb.Operations.OperationParams operation_params = 1;
     string queue_name = 2;
-    string queue_owner_aws_account_id = 3;
+    optional string queue_owner_aws_account_id = 3;
 }
 
 message GetQueueUrlResponse {
@@ -141,9 +141,9 @@ message GetQueueUrlResult {
 
 message ListQueuesRequest {
     Ydb.Operations.OperationParams operation_params = 1;
-    int64 max_results = 2;
-    string next_token = 3;
-    string queue_name_prefix = 4;
+    optional int64 max_results = 2;
+    optional string next_token = 3;
+    optional string queue_name_prefix = 4;
 }
 
 message ListQueuesResponse {
@@ -178,13 +178,13 @@ message MessageAttribute {
 message ReceiveMessageRequest {
     Ydb.Operations.OperationParams operation_params = 1;
     repeated string attribute_names = 2;
-    int32 max_number_of_messages = 3;
+    optional int32 max_number_of_messages = 3;
     repeated string message_attribute_names = 4;
     repeated string message_system_attribute_names = 5;
     string queue_url = 6;
-    string receive_request_attempt_id = 7;
-    int32 visibility_timeout = 8;
-    int32 wait_time_seconds = 9;
+    optional string receive_request_attempt_id = 7;
+    optional int32 visibility_timeout = 8;
+    optional int32 wait_time_seconds = 9;
 }
 
 message ReceiveMessageResponse {
@@ -207,11 +207,11 @@ message ReceiveMessageResult {
 
 message SendMessageRequest {
     Ydb.Operations.OperationParams operation_params = 1;
-    int32 delay_seconds = 2;
+    optional int32 delay_seconds = 2;
     map<string, MessageAttribute> message_attributes = 3;
     string message_body = 4;
-    string message_deduplication_id = 5;
-    string message_group_id = 6;
+    optional string message_deduplication_id = 5;
+    optional string message_group_id = 6;
     map<string, MessageAttribute> message_system_attributes = 7;
     string queue_url = 8;
 }
@@ -228,31 +228,73 @@ message SendMessageResult {
     string sequence_number = 5;
 }
 
-message SendMessageBatchRequest {
-    Ydb.Operations.OperationParams operation_params = 1;
-    repeated SendMessageRequest entries = 2;
-}
-
-message SendMessageBatchResponse {
-    Ydb.Operations.Operation operation = 1;
-}
-
 message BatchResultErrorEntry {
     string code = 1;
     string id = 2;
     bool sender_fault = 3;
     string message = 4;
 }
 
+message SendMessageBatchRequestEntry {
+    string id = 1;
+    optional int32 delay_seconds = 2;
+    map<string, MessageAttribute> message_attributes = 3;
+    string message_body = 4;
+    optional string message_deduplication_id = 5;
+    optional string message_group_id = 6;
+    map<string, MessageAttribute> message_system_attributes = 7;
+    string queue_url = 8;
+}
+
 message SendMessageBatchResultEntry {
-    string md5_of_message_attributes = 1;
-    string md5_of_message_body= 2;
-    string md5_of_message_system_attributes= 3;
-    string message_id = 4;
-    string sequence_number = 5;
+    string id = 1;
+    string md5_of_message_body = 2;
+    string message_id = 3;
+    string md5_of_message_attributes = 4;
+    string md5_of_message_system_attributes = 5;
+    string sequence_number = 6;
+}
+
+message SendMessageBatchRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+    repeated SendMessageBatchRequestEntry entries = 2;
+    string queue_url = 3;
+}
+
+message SendMessageBatchResponse {
+    Ydb.Operations.Operation operation = 1;
 }
 
 message SendMessageBatchResult {
     repeated BatchResultErrorEntry failed = 1;
     repeated SendMessageBatchResultEntry successful = 2;
 }
+
+message SetQueueAttributesRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+    map<string, string> attributes = 2;
+    string queue_url = 3;
+}
+
+message SetQueueAttributesResponse {
+    Ydb.Operations.Operation operation = 1;
+}
+
+message SetQueueAttributesResult {
+}
+
+message ListDeadLetterSourceQueuesRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+    optional int32 max_results = 2;
+    optional string next_token = 3;
+    string queue_url = 4;
+}
+
+message ListDeadLetterSourceQueuesResponse {
+    Ydb.Operations.Operation operation = 1;
+}
+
+message ListDeadLetterSourceQueuesResult {
+    string next_token = 1;
+    repeated string queue_urls = 2;
+}

src/api/protos/ydb_cms.proto

@@ -4,7 +4,9 @@ option cc_enable_arenas = true;
 package Ydb.Cms;
 option java_package = "com.yandex.ydb.cms";
 
+import "src/api/protos/ydb_issue_message.proto";
 import "src/api/protos/ydb_operation.proto";
+import "src/api/protos/ydb_status_codes.proto";
 
 // A set of uniform storage units.
 // Single storage unit can be thought of as a reserved part of a RAID.
@@ -104,6 +106,23 @@ message DatabaseQuotas {
     repeated StorageQuotas storage_quotas = 6;
 }
 
+// A policy that is used for resource scale recommendation. If multiple are used,
+// recommender combines them to recommend the largest scale.
+message ScaleRecommenderPolicy {
+    // Policy that tracks metric and reactively recommend to adjust resources scale
+    // to keep metric close to the specified target value.
+    message TargetTrackingPolicy {
+        oneof target {
+            // A percentage of compute resources' average CPU utilization.
+            uint32 average_cpu_utilization_percent = 1; 
+        }
+    }
+
+    oneof policy {
+        TargetTrackingPolicy target_tracking_policy = 1;
+    }
+}
+
 // Request to create a new database. For successfull creation
 // specified database shouldn't exist. At least one storage
 // unit should be requested for the database.
@@ -129,6 +148,8 @@ message CreateDatabaseRequest {
     string idempotency_key = 9;
     // Optional quotas for the database
     DatabaseQuotas database_quotas = 10;
+    // Optional scale recommender policies
+    repeated ScaleRecommenderPolicy scale_recommender_policies = 11;
 }
 
 message CreateDatabaseResponse {
@@ -179,6 +200,8 @@ message GetDatabaseStatusResult {
     SchemaOperationQuotas schema_operation_quotas = 9;
     // Current quotas for the database
     DatabaseQuotas database_quotas = 10;
+    // Current scale recommender policies
+    repeated ScaleRecommenderPolicy scale_recommender_policies = 11;
 }
 
 // Change resources allocated for database.
@@ -207,6 +230,8 @@ message AlterDatabaseRequest {
     DatabaseQuotas database_quotas = 11;
     // Alter attributes. Leave the value blank to drop an attribute.
     map<string, string> alter_attributes = 12;
+    // Alter scale recommender policies.
+    repeated ScaleRecommenderPolicy scale_recommender_policies = 13;
 }
 
 message AlterDatabaseResponse {
@@ -271,3 +296,16 @@ message DescribeDatabaseOptionsResult {
     repeated AvailabilityZoneDescription availability_zones = 2;
     repeated ComputationalUnitDescription computational_units = 3;
 }
+
+// Get resources scale recommendation for database.
+message GetScaleRecommendationRequest {
+    // Required. Full path to database's home dir.
+    string path = 1;
+}
+
+message GetScaleRecommendationResponse {
+    StatusIds.StatusCode status = 1;
+    repeated Ydb.Issue.IssueMessage issues = 2;
+    // Recommended resources scale to be allocated for database.
+    Resources recommended_resources = 3;
+}

src/api/protos/ydb_import.proto

@@ -42,7 +42,8 @@ message ImportFromS3Settings {
            The object name begins with 'source_prefix'.
            This prefix is followed by:
              * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
-             * '/scheme.pb' - object with information about scheme, indexes, etc.
+             * '/scheme.pb' - object with information about scheme, indexes, etc;
+             * '/permissions.pb' - object with information about ACL and owner.
         */
         string source_prefix = 1 [(required) = true];
 
@@ -67,6 +68,10 @@ message ImportFromS3Settings {
     // details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
     // it is especially useful for custom s3 implementations
     bool disable_virtual_addressing = 10;
+
+    // Prevent importing of ACL and owner. If true, objects are created with empty ACL
+    // and their owner will be the user who started the import.
+    bool no_acl = 11;
 }
 
 message ImportFromS3Result {

src/client/CMakeLists.txt

@@ -28,4 +28,3 @@ add_subdirectory(table)
 add_subdirectory(topic)
 add_subdirectory(types)
 add_subdirectory(value)
-add_subdirectory(ymq)

src/client/federated_topic/ut/basic_usage_ut.cpp

@@ -10,7 +10,7 @@
 #include <ydb/public/sdk/cpp/client/ydb_persqueue_public/include/write_session.h>
 
 #include <ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils/ut_utils.h>
-#include <ydb/public/sdk/cpp/client/ydb_federated_topic/ut/fds_mock.h>
+#include <ydb/public/sdk/cpp/client/ydb_federated_topic/ut/fds_mock/fds_mock.h>
 
 #include <library/cpp/testing/unittest/registar.h>
 #include <library/cpp/testing/unittest/tests_data.h>

src/client/federated_topic/ut/fds_mock.h → src/client/federated_topic/ut/fds_mock/fds_mock.h

@@ -2,7 +2,7 @@
 
 #include "util/string/builder.h"
 #include <library/cpp/threading/future/future.h>
-#include <ydb/public/api/grpc/ydb_federation_discovery_v1.grpc.pb.h>
+#include <src/api/grpc/ydb_federation_discovery_v1.grpc.pb.h>
 
 #include <deque>
 #include <optional>