Update proto files

Author: Gazizonoki

include/ydb-cpp-sdk/client/datastreams/datastreams.h

@@ -2,7 +2,7 @@
 
 #include <ydb-cpp-sdk/client/table/table.h>
 
-#include <ydb/public/api/grpc/draft/ydb_datastreams_v1.grpc.pb.h>
+#include <src/api/grpc/draft/ydb_datastreams_v1.grpc.pb.h>
 
 namespace NYdb::NDataStreams::V1 {
 

include/ydb-cpp-sdk/client/draft/ydb_scripting.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <ydb-cpp-sdk/client/table/table.h>
-#include <ydb/public/api/protos/ydb_query.pb.h>
+#include <src/api/protos/ydb_query.pb.h>
 
 namespace NYdb {
 namespace NScripting {

include/ydb-cpp-sdk/client/federated_topic/federated_topic.h

@@ -2,7 +2,7 @@
 
 #include <ydb-cpp-sdk/client/topic/client.h>
 
-#include <ydb/public/api/protos/ydb_federation_discovery.pb.h>
+#include <src/api/protos/ydb_federation_discovery.pb.h>
 
 #include <ydb-cpp-sdk/client/types/exceptions/exceptions.h>
 

include/ydb-cpp-sdk/client/proto/accessor.h

@@ -1,13 +1,13 @@
 #pragma once
 
-#include <ydb/public/api/protos/draft/ydb_replication.pb.h>
-#include <ydb/public/api/protos/ydb_coordination.pb.h>
-#include <ydb/public/api/protos/ydb_export.pb.h>
-#include <ydb/public/api/protos/ydb_import.pb.h>
-#include <ydb/public/api/protos/ydb_query_stats.pb.h>
-#include <ydb/public/api/protos/ydb_table.pb.h>
-#include <ydb/public/api/protos/ydb_topic.pb.h>
-#include <ydb/public/api/protos/ydb_value.pb.h>
+#include <src/api/protos/draft/ydb_replication.pb.h>
+#include <src/api/protos/ydb_coordination.pb.h>
+#include <src/api/protos/ydb_export.pb.h>
+#include <src/api/protos/ydb_import.pb.h>
+#include <src/api/protos/ydb_query_stats.pb.h>
+#include <src/api/protos/ydb_table.pb.h>
+#include <src/api/protos/ydb_topic.pb.h>
+#include <src/api/protos/ydb_value.pb.h>
 
 #include <ydb-cpp-sdk/client/draft/ydb_replication.h>
 #include <ydb-cpp-sdk/client/coordination/coordination.h>

include/ydb-cpp-sdk/client/topic/control_plane.h

@@ -4,7 +4,7 @@
 
 #include <ydb-cpp-sdk/client/scheme/scheme.h>
 
-#include <ydb/public/api/protos/ydb_topic.pb.h>
+#include <src/api/protos/ydb_topic.pb.h>
 
 #include <util/datetime/base.h>
 

src/api/grpc/draft/ydb_backup_v1.proto

@@ -0,0 +1,33 @@
+syntax = "proto3";
+option cc_enable_arenas = true;
+
+package Ydb.Backup.V1;
+
+option java_package = "com.yandex.ydb.backup.v1";
+
+import "src/api/protos/draft/ydb_backup.proto";
+
+service BackupService {
+    // Fetches data from remote storage to cluster instance
+    // After successful completion of this call BackupCollection objects are created inside cluster
+    // And remote data can be accessed through them
+    rpc FetchBackupCollections(FetchBackupCollectionsRequest) returns (FetchBackupCollectionsResponse) {}
+
+    // Shows BackupCollection's present in cluster (probably with some filters or accessible by current user)
+    // Exact storage scheme of this collections will be defined later (probably at the beginning it will be some flat scheme by UUID with indices for filtering)
+    rpc ListBackupCollections(ListBackupCollectionsRequest) returns (ListBackupCollectionsResponse) {}
+
+    // Creates backup collection with specified settings
+    // In case of remote backup collection (e.g. stored in s3) also checks connection to s3 and all corresponding rights
+    rpc CreateBackupCollection(CreateBackupCollectionRequest) returns (CreateBackupCollectionResponse) {}
+
+    // Returns specified BackupCollection data
+    rpc ReadBackupCollection(ReadBackupCollectionRequest) returns (ReadBackupCollectionResponse) {}
+
+    // Somehow updates BackupCollection state. Maybe all subobjects will be modified through this call, maybe they will have separate calls
+    rpc UpdateBackupCollection(UpdateBackupCollectionRequest) returns (UpdateBackupCollectionResponse) {}
+
+    // Deletes BackupCollection. In case of remote backup collection data WILL be preserved if other is not specified explicitly
+    // In case of read-only BackupCollection's additionally to this explicit flag user should set force flag.
+    rpc DeleteBackupCollection(DeleteBackupCollectionRequest) returns (DeleteBackupCollectionResponse) {}
+}

src/api/grpc/draft/ydb_object_storage_v1.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+package Ydb.ObjectStorage.V1;
+option java_package = "com.yandex.ydb.object_storage.v1";
+
+import "src/api/protos/draft/ydb_object_storage.proto";
+
+service ObjectStorageService {
+    rpc List(ObjectStorage.ListingRequest) returns (ObjectStorage.ListingResponse);
+}

src/api/grpc/draft/ydb_tablet_v1.proto

@@ -0,0 +1,34 @@
+syntax = "proto3";
+
+package Ydb.Tablet.V1;
+option java_package = "com.yandex.ydb.tablet.v1";
+
+import "src/api/protos/draft/ydb_tablet.proto";
+
+// This service has low-level APIs that are not intended for normal operation,
+// sometimes allowing system administrators to repair broken databases or
+// perform reconfiguration that is not directly supported. These may break the
+// system completely however, so use with caution and make sure you really know
+// what you're doing.
+service TabletService {
+
+    // Execute a minikql script at the specified tablet.
+    // This may be used to side-step intended tablet APIs and work with their
+    // local database directly, e.g. to repair tablets in an unexpected state.
+    // Note the modified tablet usually needs to be restarted to pick up any
+    // changes, since its logical in-memory state is not updated and diverges
+    // from the new persistent state.
+    rpc ExecuteTabletMiniKQL(ExecuteTabletMiniKQLRequest) returns (ExecuteTabletMiniKQLResponse);
+
+    // Change schema of the specified tablet and get the resulting schema.
+    // This may be used to side-step intended tablet APIs and repair tablets
+    // in an unexpected state. Empty changes are useful to dump current tablet
+    // schema.
+    rpc ChangeTabletSchema(ChangeTabletSchemaRequest) returns (ChangeTabletSchemaResponse);
+
+    // Restart the specified tablet.
+    // This may help when tablets (or their clients) are running in an
+    // unexpected state, sometimes allowing them to get unstuck.
+    rpc RestartTablet(RestartTabletRequest) returns (RestartTabletResponse);
+
+}

src/api/grpc/draft/ydb_ymq_v1.proto

@@ -5,8 +5,10 @@ package Ydb.Ymq.V1;
 
 import "src/api/protos/draft/ymq.proto";
 
+
 option java_package = "com.yandex.ydb.ymq.v1";
 
+
 service YmqService {
     rpc GetQueueUrl(GetQueueUrlRequest) returns (GetQueueUrlResponse);
     rpc CreateQueue(CreateQueueRequest) returns (CreateQueueResponse);

src/api/grpc/ydb_query_v1.proto

@@ -6,11 +6,6 @@ option java_package = "com.yandex.ydb.query.v1";
 import "src/api/protos/ydb_operation.proto";
 import "src/api/protos/ydb_query.proto";
 
-// ! WARNING: Experimental API
-// ! This API is currently in experimental state and is a subject for changes.
-// ! No backward and/or forward compatibility guarantees are provided.
-// ! DO NOT USE for production workloads.
-
 service QueryService {
     // Sessions are basic primitives for communicating with YDB Query Service. The are similar to
     // connections for classic relational DBs. Sessions serve three main purposes:

src/api/grpc/ydb_topic_v1.proto

@@ -96,15 +96,19 @@ service TopicService {
     // Create DirectRead Session
     // Pipeline:
     // client                  server
-    //         StartDirectReadPartitionSession(SessionID1, PartitionSessionID1, TabletGeneration1, Secret1,...)
+    //         InitRequest(SessionID1, TopicReadSettings, Consumer)
     //        ---------------->
+    //         InitResponse
+    //        <----------------
+    //         StartDirectReadPartitionSessionRequest(PartitionSessionID1, LastDirectReadID, TabletGeneration1)
+    //        ---------------->
+    //         StartDirectReadPartitionSessionResponse(PartitionSessionID1, TabletGeneration1)
+    //        <----------------
     //         DirectReadResponse(PartitionSessionID1, DirectReadID1, Secret1, ...)
     //        <----------------
     //         DirectReadResponse(PartitionSessionID1, DirectReadID2, Secret1, ...)
     //        <----------------
-    //         UpdateDirectReadPartitionSession(SessionID1, PartitionSessionID1, TabletGeneration1, Secret1,...)
-    //        ---------------->
-    //         StopDirectReadPartitionSession(SessionID1, PartitionSessionID1, TabletGeneration1, Secret1,...)
+    //         StopDirectReadPartitionSession(status, issues, PartitionSessionID1, TabletGeneration1)
     //        <----------------
     rpc StreamDirectRead(stream StreamDirectReadMessage.FromClient) returns (stream StreamDirectReadMessage.FromServer);
 

src/api/protos/annotations/ya.make

@@ -0,0 +1,10 @@
+PROTO_LIBRARY()
+
+SRCS(
+    sensitive.proto
+    validation.proto
+)
+
+EXCLUDE_TAGS(GO_PROTO)
+
+END()

src/api/protos/draft/datastreams.proto

@@ -5,6 +5,7 @@ import "src/api/protos/ydb_operation.proto";
 import "src/api/protos/draft/field_transformation.proto";
 
 import "google/protobuf/descriptor.proto";
+import "google/protobuf/duration.proto";
 
 package Ydb.DataStreams.V1;
 option java_package = "com.yandex.ydb.datastreams.v1";
@@ -110,6 +111,8 @@ message StreamDescription {
 
     // stream metering mode
     StreamModeDetails stream_mode_details = 14;
+
+    PartitioningSettings partitioning_settings = 15;
 }
 
 // Represents range of possible sequence numbers for the shard
@@ -264,6 +267,55 @@ message StreamModeDetails {
     StreamMode stream_mode = 1;
 }
 
+enum AutoPartitioningStrategy {
+    // The auto partitioning algorithm is not specified. The default value will be used.
+    AUTO_PARTITIONING_STRATEGY_UNSPECIFIED = 0;
+    // The auto partitioning is disabled.
+    AUTO_PARTITIONING_STRATEGY_DISABLED = 1;
+    // The auto partitioning algorithm will increase partitions count depending on the load characteristics.
+    // The auto partitioning algorithm will never decrease the number of partitions.
+    AUTO_PARTITIONING_STRATEGY_SCALE_UP = 2;
+    // The auto partitioning algorithm will both increase and decrease partitions count depending on the load characteristics.
+    AUTO_PARTITIONING_STRATEGY_SCALE_UP_AND_DOWN = 3;
+    // The auto partitioning is paused.
+    AUTO_PARTITIONING_STRATEGY_PAUSED = 4;
+}
+
+// Partitioning settings for stream.
+message PartitioningSettings {
+    // Auto merge would stop working when the partitions count reaches min_active_partitions
+    // Zero value means default - 1.
+    int64 min_active_partitions = 1;
+    // Auto split would stop working when the partitions count reaches max_active_partitions
+    // Zero value means default - 1.
+    int64 max_active_partitions = 3;
+    // Settings for the partitions count auto partitioning.
+    AutoPartitioningSettings auto_partitioning_settings = 4;
+}
+
+message AutoPartitioningSettings {
+    // Strategy of auto partitioning.
+    AutoPartitioningStrategy strategy = 1;
+    // Partition write speed auto partitioning options.
+    AutoPartitioningWriteSpeedStrategy partition_write_speed = 2;
+}
+
+message AutoPartitioningWriteSpeedStrategy {
+    //Partition will be auto partitioned up (divided into 2 partitions)
+    //after write speed to the partition exceeds up_utilization_percent (in percentage of maximum write speed to the partition) for the period of time stabilization_window
+
+    //Partition will become a candidate to the auto partitioned down
+    //after write speed doesn’t reach down_utilization_percent (in percentage of maximum write speed to the partition) for the period of time stabilization_window
+    //This candidate partition will be auto partitioned down when other neighbour partition will become a candidate to the auto partitioning down and not earlier than a retention period.
+
+    // Zero value means default - 300.
+    google.protobuf.Duration stabilization_window = 1;
+    // Zero value means default - 90.
+    int32 up_utilization_percent = 2;
+    // Zero value means default - 30.
+    int32 down_utilization_percent = 3;
+}
+
 message CreateStreamRequest {
     Ydb.Operations.OperationParams operation_params = 1;
     // Name of the stream
@@ -280,6 +332,8 @@ message CreateStreamRequest {
     }
     // stream metering mode
     StreamModeDetails stream_mode_details = 7;
+
+    PartitioningSettings partitioning_settings = 8;
 }
 
 message CreateStreamResponse {
@@ -387,6 +441,7 @@ message UpdateStreamRequest {
     // stream metering mode
     StreamModeDetails stream_mode_details = 7;
 
+    PartitioningSettings partitioning_settings = 8;
 }
 
 message UpdateStreamResponse {
@@ -622,7 +677,7 @@ message AddTagsToStreamRequest {
     // Name of the stream
     string stream_name = 2;
     // List of tags to add to stream
-    repeated string tags = 3;
+    map<string, string> tags = 3;
 }
 
 message AddTagsToStreamResponse {
@@ -899,4 +954,3 @@ message UpdateStreamModeResponse {
 
 message UpdateStreamModeResult {
 }
-

src/api/protos/draft/fq.proto

@@ -19,7 +19,7 @@ import "google/protobuf/empty.proto";
 
 // === Query API ===
 
-// Header: ydb-fq-project => yandexcloud://cloud_id/folder_id
+// Header: ydb-fq-project => yandexcloud://folder_id
 
 message Acl {
     enum Visibility {
@@ -126,8 +126,10 @@ message QueryContent {
     // "executor" - type of executor for this query
     map<string, string> execution_settings = 10 [(Ydb.map_key).length.range = {min: 1, max: 100}, (Ydb.length).le = 4096];
     // Syntax of the text
-    // By default it is "yql", but additional syntax "pg" (PostreSQL SQL frontend) is supported as well
+    // By default it is "yql", but additional syntax "pg" (PostgreSQL SQL frontend) is supported as well
     QuerySyntax syntax = 11;
+    // Map of query parameters
+    map<string, Ydb.TypedValue> parameters = 12 [(Ydb.map_key).length.range = {min: 1, max: 100}];
 }
 
 message CommonMeta {
@@ -372,6 +374,7 @@ message Job {
     bool automatic = 11;
     google.protobuf.Timestamp expire_at = 12;
     QueryContent.QuerySyntax syntax = 13;
+    map<string, Ydb.TypedValue> parameters = 14;
 }
 
 // Information about recent query runs
@@ -489,6 +492,23 @@ message PostgreSQLCluster {
     bool secure = 7;
 }
 
+message GreenplumCluster {
+    string database_id = 1 [(Ydb.length).le = 1024];
+    string database_name = 2 [(Ydb.length).le = 1024];
+    string login = 3 [(Ydb.length).le = 1024, (Ydb.sensitive) = true];
+    string password = 4 [(Ydb.length).le = 1024, (Ydb.sensitive) = true];
+    string schema = 5 [(Ydb.length).le = 1024];
+    IamAuth auth = 6;
+}
+
+message MySQLCluster {
+    string database_id = 1 [(Ydb.length).le = 1024];
+    string database_name = 2 [(Ydb.length).le = 1024];
+    string login = 3 [(Ydb.length).le = 1024, (Ydb.sensitive) = true];
+    string password = 4 [(Ydb.length).le = 1024, (Ydb.sensitive) = true];
+    IamAuth auth = 5;
+}
+
 message ConnectionSetting {
     enum ConnectionType {
         CONNECTION_TYPE_UNSPECIFIED = 0;
@@ -498,6 +518,8 @@ message ConnectionSetting {
         OBJECT_STORAGE = 4;
         MONITORING = 5;
         POSTGRESQL_CLUSTER = 6;
+        GREENPLUM_CLUSTER = 7;
+        MYSQL_CLUSTER = 8;
     }
 
     oneof connection {
@@ -507,6 +529,8 @@ message ConnectionSetting {
         ObjectStorageConnection object_storage = 4;
         Monitoring monitoring = 5;
         PostgreSQLCluster postgresql_cluster = 6;
+        GreenplumCluster greenplum_cluster = 7;
+        MySQLCluster mysql_cluster = 8;
     }
 }
 

src/api/protos/draft/persqueue_common.proto

@@ -2,6 +2,8 @@ syntax = "proto3";
 import "google/protobuf/descriptor.proto";
 import "src/api/protos/draft/persqueue_error_codes.proto";
 
+import "src/api/protos/annotations/sensitive.proto";
+
 package NPersQueueCommon;
 
 option java_package = "com.yandex.ydb.persqueue";
@@ -35,8 +37,7 @@ enum ECodec {
 
 message Credentials {
     oneof credentials {
-        bytes tvm_service_ticket = 1;
-        bytes oauth_token = 2;
+        bytes tvm_service_ticket = 1 [(Ydb.sensitive) = true];
+        bytes oauth_token = 2 [(Ydb.sensitive) = true];
     }
 }
-

src/api/protos/draft/persqueue_error_codes.proto

@@ -12,6 +12,7 @@ enum EErrorCode {
     SOURCEID_DELETED = 24;
 
     WRITE_ERROR_PARTITION_IS_FULL = 5;
+    WRITE_ERROR_PARTITION_INACTIVE = 29;
     WRITE_ERROR_DISK_IS_FULL = 15;
     WRITE_ERROR_BAD_OFFSET = 19;