Skip to content

Commit 7b2c0d2

Browse files
shnikdshnikd
authored
Support ydb dump for tables with serial types (#9272)
1 parent a10501d commit 7b2c0d2

File tree

10 files changed

+217
-27
lines changed

10 files changed

+217
-27
lines changed

‎ydb/core/grpc_services/rpc_create_table.cpp

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,28 @@ class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreate
181181
return;
182182
}
183183

184+
StatusIds::StatusCode code = StatusIds::SUCCESS;
185+
TString error;
186+
187+
bool hasSerial = false;
188+
for (const auto& column : req->columns()) {
189+
switch (column.default_value_case()) {
190+
case Ydb::Table::ColumnMeta::kFromSequence: {
191+
auto* seqDesc = modifyScheme->MutableCreateIndexedTable()->MutableSequenceDescription()->Add();
192+
if (!FillSequenceDescription(*seqDesc, column.from_sequence(), code, error)) {
193+
NYql::TIssues issues;
194+
issues.AddIssue(NYql::TIssue(error));
195+
return Reply(code, issues, ctx);
196+
}
197+
hasSerial = true;
198+
break;
199+
}
200+
default: break;
201+
}
202+
}
203+
184204
NKikimrSchemeOp::TTableDescription* tableDesc = nullptr;
185-
if (req->indexesSize()) {
205+
if (req->indexesSize() || hasSerial) {
186206
modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable);
187207
tableDesc = modifyScheme->MutableCreateIndexedTable()->MutableTableDescription();
188208
} else {
@@ -192,9 +212,6 @@ class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreate
192212

193213
tableDesc->SetName(name);
194214

195-
StatusIds::StatusCode code = StatusIds::SUCCESS;
196-
TString error;
197-
198215
if (!FillColumnDescription(*tableDesc, req->columns(), code, error)) {
199216
NYql::TIssues issues;
200217
issues.AddIssue(NYql::TIssue(error));

‎ydb/core/grpc_services/rpc_describe_table.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,14 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe
143143
return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
144144
}
145145

146+
StatusIds::StatusCode code = StatusIds::SUCCESS;
147+
TString error;
148+
if (!FillSequenceDescription(describeTableResult, tableDescription, code, error)) {
149+
LOG_ERROR(ctx, NKikimrServices::GRPC_SERVER, "Unable to fill sequence description: %s", error.c_str());
150+
Request_->RaiseIssue(NYql::TIssue(error));
151+
return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
152+
}
153+
146154
describeTableResult.mutable_primary_key()->CopyFrom(tableDescription.GetKeyColumnNames());
147155

148156
try {
@@ -213,6 +221,10 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe
213221
record->MutableOptions()->SetReturnPartitionStats(true);
214222
}
215223

224+
if (req->include_set_val()) {
225+
record->MutableOptions()->SetReturnSetVal(true);
226+
}
227+
216228
record->MutableOptions()->SetShowPrivateTable(ShowPrivatePath(path));
217229

218230
ctx.Send(MakeTxProxyID(), navigateRequest.release());

‎ydb/core/ydb_convert/table_description.cpp

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1425,7 +1425,6 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out,
14251425
const Ydb::Table::CreateTableRequest& in, const TTableProfiles& profiles,
14261426
Ydb::StatusIds::StatusCode& status, TString& error, bool indexedTable)
14271427
{
1428-
14291428
NKikimrSchemeOp::TTableDescription* tableDesc = nullptr;
14301429
if (indexedTable) {
14311430
tableDesc = out.MutableCreateIndexedTable()->MutableTableDescription();
@@ -1467,7 +1466,8 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out,
14671466
return true;
14681467
}
14691468

1470-
bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
1469+
template <typename TYdbProto>
1470+
bool FillSequenceDescriptionImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
14711471
THashMap<TString, NKikimrSchemeOp::TSequenceDescription> sequences;
14721472

14731473
for (const auto& sequenceDescription : in.GetSequences()) {
@@ -1568,6 +1568,14 @@ bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrS
15681568
return true;
15691569
}
15701570

1571+
bool FillSequenceDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
1572+
return FillSequenceDescriptionImpl(out, in, status, error);
1573+
}
1574+
1575+
bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
1576+
return FillSequenceDescriptionImpl(out, in, status, error);
1577+
}
1578+
15711579
bool FillSequenceDescription(NKikimrSchemeOp::TSequenceDescription& out, const Ydb::Table::SequenceDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
15721580
out.SetName(in.name());
15731581
if (in.has_min_value()) {

‎ydb/core/ydb_convert/table_description.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,9 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out,
133133

134134

135135
// out
136-
bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in,
137-
Ydb::StatusIds::StatusCode& status, TString& error);
136+
bool FillSequenceDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error);
137+
138+
bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error);
138139

139140
// in
140141
bool FillSequenceDescription(

‎ydb/library/backup/backup.cpp

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ NTable::TTableDescription DescribeTable(TDriver driver, const TString& fullTable
374374
NTable::TTableClient client(driver);
375375

376376
TStatus status = client.RetryOperationSync([fullTablePath, &desc](NTable::TSession session) {
377-
auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true);
377+
auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true).WithSetVal(true);
378378
auto result = session.DescribeTable(fullTablePath, settings).GetValueSync();
379379

380380
VerifyStatus(result);
@@ -514,6 +514,7 @@ void BackupTable(TDriver driver, const TString& dbPrefix, const TString& backupP
514514
<< " backupPrefix: " << backupPrefix << " path: " << path);
515515

516516
auto desc = DescribeTable(driver, JoinDatabasePath(schemaOnly ? dbPrefix : backupPrefix, path));
517+
517518
auto proto = ProtoFromTableDescription(desc, preservePoolKinds);
518519

519520
TString schemaStr;
@@ -742,19 +743,26 @@ void BackupFolder(TDriver driver, const TString& database, const TString& relDbP
742743
// Restore
743744
////////////////////////////////////////////////////////////////////////////////
744745

745-
TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTableBuilder *builder) {
746+
TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTableBuilder *builder, std::optional<NTable::TSequenceDescription> sequenceDescription) {
746747
TStringStream ss;
747748
ss << "name: " << name << "; ";
748749
if (parser.GetKind() == TTypeParser::ETypeKind::Optional) {
749750
ss << " optional; ";
750751
parser.OpenOptional();
751752
}
753+
if (sequenceDescription.has_value()) {
754+
ss << "serial; ";
755+
}
752756
ss << "kind: " << parser.GetKind() << "; ";
753757
switch (parser.GetKind()) {
754758
case TTypeParser::ETypeKind::Primitive:
755759
ss << " type_id: " << parser.GetPrimitive() << "; ";
756760
if (builder) {
757-
builder->AddNullableColumn(name, parser.GetPrimitive());
761+
if (sequenceDescription.has_value()) {
762+
builder->AddSerialColumn(name, parser.GetPrimitive(), std::move(*sequenceDescription));
763+
} else {
764+
builder->AddNullableColumn(name, parser.GetPrimitive());
765+
}
758766
}
759767
break;
760768
case TTypeParser::ETypeKind::Decimal:
@@ -775,8 +783,19 @@ TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTabl
775783
NTable::TTableDescription TableDescriptionFromProto(const Ydb::Table::CreateTableRequest& proto) {
776784
NTable::TTableBuilder builder;
777785

786+
std::optional<NTable::TSequenceDescription> sequenceDescription;
778787
for (const auto &col : proto.Getcolumns()) {
779-
LOG_DEBUG("AddNullableColumn: " << ProcessColumnType(col.Getname(), TType(col.Gettype()), &builder));
788+
if (col.from_sequence().name() == "_serial_column_" + col.name()) {
789+
NTable::TSequenceDescription currentSequenceDescription;
790+
if (col.from_sequence().has_set_val()) {
791+
NTable::TSequenceDescription::TSetVal setVal;
792+
setVal.NextUsed = col.from_sequence().set_val().next_used();
793+
setVal.NextValue = col.from_sequence().set_val().next_value();
794+
currentSequenceDescription.SetVal = std::move(setVal);
795+
}
796+
sequenceDescription = std::move(currentSequenceDescription);
797+
}
798+
LOG_DEBUG("AddColumn: " << ProcessColumnType(col.Getname(), TType(col.Gettype()), &builder, std::move(sequenceDescription)));
780799
}
781800

782801
for (const auto &primary : proto.Getprimary_key()) {
@@ -805,7 +824,7 @@ TString SerializeColumnsToString(const TVector<TColumn>& columns, TVector<TStrin
805824
if (BinarySearch(primary.cbegin(), primary.cend(), col.Name)) {
806825
ss << "primary; ";
807826
}
808-
ss << ProcessColumnType(col.Name, col.Type, nullptr) << Endl;
827+
ss << ProcessColumnType(col.Name, col.Type, nullptr, std::nullopt) << Endl;
809828
}
810829
// Cerr << "Parse column to : " << ss.Str() << Endl;
811830
return ss.Str();

‎ydb/public/api/protos/ydb_table.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -779,6 +779,8 @@ message DescribeTableRequest {
779779
bool include_table_stats = 6;
780780
// Includes partition statistics (required include_table_statistics)
781781
bool include_partition_stats = 7;
782+
// Includes set_val settings for sequences
783+
bool include_set_val = 8;
782784
}
783785

784786
message DescribeTableResponse {

‎ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,10 @@ TAsyncDescribeTableResult TTableClient::TImpl::DescribeTable(const TString& sess
525525
request.set_include_partition_stats(true);
526526
}
527527

528+
if (settings.WithSetVal_) {
529+
request.set_include_set_val(true);
530+
}
531+
528532
auto promise = NewPromise<TDescribeTableResult>();
529533

530534
auto extractor = [promise, settings]

‎ydb/public/sdk/cpp/client/ydb_table/table.cpp

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,24 @@ class TTableDescription::TImpl {
291291
if (col.has_not_null()) {
292292
not_null = col.not_null();
293293
}
294-
Columns_.emplace_back(col.name(), col.type(), col.family(), not_null);
294+
std::optional<TSequenceDescription> sequenceDescription;
295+
switch (col.default_value_case()) {
296+
case Ydb::Table::ColumnMeta::kFromSequence: {
297+
if (col.from_sequence().name() == "_serial_column_" + col.name()) {
298+
TSequenceDescription currentSequenceDescription;
299+
if (col.from_sequence().has_set_val()) {
300+
TSequenceDescription::TSetVal setVal;
301+
setVal.NextUsed = col.from_sequence().set_val().next_used();
302+
setVal.NextValue = col.from_sequence().set_val().next_value();
303+
currentSequenceDescription.SetVal = std::move(setVal);
304+
}
305+
sequenceDescription = std::move(currentSequenceDescription);
306+
}
307+
break;
308+
}
309+
default: break;
310+
}
311+
Columns_.emplace_back(col.name(), col.type(), col.family(), not_null, std::move(sequenceDescription));
295312
}
296313

297314
// indexes
@@ -453,8 +470,8 @@ class TTableDescription::TImpl {
453470
return Proto_;
454471
}
455472

456-
void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull) {
457-
Columns_.emplace_back(name, type, family, notNull);
473+
void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull, std::optional<TSequenceDescription> sequenceDescription) {
474+
Columns_.emplace_back(name, type, family, notNull, std::move(sequenceDescription));
458475
}
459476

460477
void SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns) {
@@ -737,8 +754,8 @@ const TVector<TKeyRange>& TTableDescription::GetKeyRanges() const {
737754
return Impl_->GetKeyRanges();
738755
}
739756

740-
void TTableDescription::AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull) {
741-
Impl_->AddColumn(name, type, family, notNull);
757+
void TTableDescription::AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull, std::optional<TSequenceDescription> sequenceDescription) {
758+
Impl_->AddColumn(name, type, family, notNull, std::move(sequenceDescription));
742759
}
743760

744761
void TTableDescription::SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns) {
@@ -914,6 +931,15 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con
914931
if (column.NotNull.has_value()) {
915932
protoColumn.set_not_null(column.NotNull.value());
916933
}
934+
if (column.SequenceDescription.has_value()) {
935+
auto* fromSequence = protoColumn.mutable_from_sequence();
936+
if (column.SequenceDescription->SetVal.has_value()) {
937+
auto* setVal = fromSequence->mutable_set_val();
938+
setVal->set_next_value(column.SequenceDescription->SetVal->NextValue);
939+
setVal->set_next_used(column.SequenceDescription->SetVal->NextUsed);
940+
}
941+
fromSequence->set_name("_serial_column_" + column.Name);
942+
}
917943
}
918944

919945
for (const auto& pk : Impl_->GetPrimaryKeyColumns()) {
@@ -1121,7 +1147,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const EPrim
11211147
.EndOptional()
11221148
.Build();
11231149

1124-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false);
1150+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt);
11251151
return *this;
11261152
}
11271153

@@ -1131,7 +1157,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TDeci
11311157
.Decimal(type)
11321158
.EndOptional()
11331159
.Build();
1134-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false);
1160+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt);
11351161
return *this;
11361162
}
11371163

@@ -1140,7 +1166,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TPgTy
11401166
.Pg(type)
11411167
.Build();
11421168

1143-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false);
1169+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt);
11441170
return *this;
11451171
}
11461172

@@ -1149,7 +1175,7 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const EP
11491175
.Primitive(type)
11501176
.Build();
11511177

1152-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true);
1178+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt);
11531179
return *this;
11541180
}
11551181

@@ -1158,7 +1184,7 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TD
11581184
.Decimal(type)
11591185
.Build();
11601186

1161-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true);
1187+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt);
11621188
return *this;
11631189
}
11641190

@@ -1167,7 +1193,16 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TP
11671193
.Pg(type)
11681194
.Build();
11691195

1170-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true);
1196+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt);
1197+
return *this;
1198+
}
1199+
1200+
TTableBuilder& TTableBuilder::AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family) {
1201+
auto columnType = TTypeBuilder()
1202+
.Primitive(type)
1203+
.Build();
1204+
1205+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::move(sequenceDescription));
11711206
return *this;
11721207
}
11731208

‎ydb/public/sdk/cpp/client/ydb_table/table.h

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,19 +100,29 @@ class TKeyRange {
100100
TMaybe<TKeyBound> To_;
101101
};
102102

103+
struct TSequenceDescription {
104+
struct TSetVal {
105+
i64 NextValue;
106+
bool NextUsed;
107+
};
108+
std::optional<TSetVal> SetVal;
109+
};
110+
103111
struct TTableColumn {
104112
TString Name;
105113
TType Type;
106114
TString Family;
107115
std::optional<bool> NotNull;
116+
std::optional<TSequenceDescription> SequenceDescription;
108117

109118
TTableColumn() = default;
110119

111-
TTableColumn(TString name, TType type, TString family = TString(), std::optional<bool> notNull = std::nullopt)
120+
TTableColumn(TString name, TType type, TString family = TString(), std::optional<bool> notNull = std::nullopt, std::optional<TSequenceDescription> sequenceDescription = std::nullopt)
112121
: Name(std::move(name))
113122
, Type(std::move(type))
114123
, Family(std::move(family))
115124
, NotNull(std::move(notNull))
125+
, SequenceDescription(std::move(sequenceDescription))
116126
{ }
117127

118128
// Conversion from TColumn for API compatibility
@@ -636,7 +646,7 @@ class TTableDescription {
636646
TTableDescription();
637647
explicit TTableDescription(const Ydb::Table::CreateTableRequest& request);
638648

639-
void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull);
649+
void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull, std::optional<TSequenceDescription> sequenceDescription);
640650
void SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns);
641651

642652
// common
@@ -854,6 +864,7 @@ class TTableBuilder {
854864
TTableBuilder& AddNonNullableColumn(const TString& name, const TPgType& type, const TString& family = TString());
855865
TTableBuilder& SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns);
856866
TTableBuilder& SetPrimaryKeyColumn(const TString& primaryKeyColumn);
867+
TTableBuilder& AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family = TString());
857868

858869
// common
859870
TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription);
@@ -1629,6 +1640,7 @@ struct TDescribeTableSettings : public TOperationRequestSettings<TDescribeTableS
16291640
FLUENT_SETTING_DEFAULT(bool, WithKeyShardBoundary, false);
16301641
FLUENT_SETTING_DEFAULT(bool, WithTableStatistics, false);
16311642
FLUENT_SETTING_DEFAULT(bool, WithPartitionStatistics, false);
1643+
FLUENT_SETTING_DEFAULT(bool, WithSetVal, false);
16321644
};
16331645

16341646
struct TExplainDataQuerySettings : public TOperationRequestSettings<TExplainDataQuerySettings> {

0 commit comments

Comments
 (0)