Skip to content

Commit 9809a3c

Browse files
shnikduzhastik
shnikd
authored and
uzhastik
committed
Support ydb dump for tables with serial types (#9757)
1 parent 429745b commit 9809a3c

File tree

11 files changed

+237
-31
lines changed

11 files changed

+237
-31
lines changed

‎ydb/core/grpc_services/rpc_create_table.cpp

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,28 @@ class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreate
181181
return;
182182
}
183183

184+
StatusIds::StatusCode code = StatusIds::SUCCESS;
185+
TString error;
186+
187+
bool hasSerial = false;
188+
for (const auto& column : req->columns()) {
189+
switch (column.default_value_case()) {
190+
case Ydb::Table::ColumnMeta::kFromSequence: {
191+
auto* seqDesc = modifyScheme->MutableCreateIndexedTable()->MutableSequenceDescription()->Add();
192+
if (!FillSequenceDescription(*seqDesc, column.from_sequence(), code, error)) {
193+
NYql::TIssues issues;
194+
issues.AddIssue(NYql::TIssue(error));
195+
return Reply(code, issues, ctx);
196+
}
197+
hasSerial = true;
198+
break;
199+
}
200+
default: break;
201+
}
202+
}
203+
184204
NKikimrSchemeOp::TTableDescription* tableDesc = nullptr;
185-
if (req->indexesSize()) {
205+
if (req->indexesSize() || hasSerial) {
186206
modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable);
187207
tableDesc = modifyScheme->MutableCreateIndexedTable()->MutableTableDescription();
188208
} else {
@@ -192,9 +212,6 @@ class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreate
192212

193213
tableDesc->SetName(name);
194214

195-
StatusIds::StatusCode code = StatusIds::SUCCESS;
196-
TString error;
197-
198215
if (!FillColumnDescription(*tableDesc, req->columns(), code, error)) {
199216
NYql::TIssues issues;
200217
issues.AddIssue(NYql::TIssue(error));

‎ydb/core/grpc_services/rpc_describe_table.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,14 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe
142142
return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
143143
}
144144

145+
StatusIds::StatusCode code = StatusIds::SUCCESS;
146+
TString error;
147+
if (!FillSequenceDescription(describeTableResult, tableDescription, code, error)) {
148+
LOG_ERROR(ctx, NKikimrServices::GRPC_SERVER, "Unable to fill sequence description: %s", error.c_str());
149+
Request_->RaiseIssue(NYql::TIssue(error));
150+
return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
151+
}
152+
145153
describeTableResult.mutable_primary_key()->CopyFrom(tableDescription.GetKeyColumnNames());
146154

147155
try {
@@ -212,6 +220,10 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe
212220
record->MutableOptions()->SetReturnPartitionStats(true);
213221
}
214222

223+
if (req->include_set_val()) {
224+
record->MutableOptions()->SetReturnSetVal(true);
225+
}
226+
215227
record->MutableOptions()->SetShowPrivateTable(ShowPrivatePath(path));
216228

217229
ctx.Send(MakeTxProxyID(), navigateRequest.release());

‎ydb/core/tx/datashard/export_common.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,12 @@ TMaybe<Ydb::Table::CreateTableRequest> GenYdbScheme(
6666
FillPartitioningSettings(scheme, tableDesc);
6767
FillKeyBloomFilter(scheme, tableDesc);
6868
FillReadReplicasSettings(scheme, tableDesc);
69-
FillSequenceDescription(scheme, tableDesc);
69+
70+
TString error;
71+
Ydb::StatusIds::StatusCode status;
72+
if (!FillSequenceDescription(scheme, tableDesc, status, error)) {
73+
return Nothing();
74+
}
7075

7176
return scheme;
7277
}

‎ydb/core/ydb_convert/table_description.cpp

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1390,7 +1390,6 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out,
13901390
const Ydb::Table::CreateTableRequest& in, const TTableProfiles& profiles,
13911391
Ydb::StatusIds::StatusCode& status, TString& error, bool indexedTable)
13921392
{
1393-
13941393
NKikimrSchemeOp::TTableDescription* tableDesc = nullptr;
13951394
if (indexedTable) {
13961395
tableDesc = out.MutableCreateIndexedTable()->MutableTableDescription();
@@ -1432,7 +1431,10 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out,
14321431
return true;
14331432
}
14341433

1435-
void FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in) {
1434+
template <typename TYdbProto>
1435+
bool FillSequenceDescriptionImpl(TYdbProto& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
1436+
Y_UNUSED(status);
1437+
Y_UNUSED(error);
14361438
THashMap<TString, NKikimrSchemeOp::TSequenceDescription> sequences;
14371439

14381440
for (const auto& sequenceDescription : in.GetSequences()) {
@@ -1478,6 +1480,45 @@ void FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrS
14781480
default: break;
14791481
}
14801482
}
1483+
return true;
1484+
}
1485+
1486+
bool FillSequenceDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
1487+
return FillSequenceDescriptionImpl(out, in, status, error);
1488+
}
1489+
1490+
bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
1491+
return FillSequenceDescriptionImpl(out, in, status, error);
1492+
}
1493+
1494+
bool FillSequenceDescription(NKikimrSchemeOp::TSequenceDescription& out, const Ydb::Table::SequenceDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) {
1495+
Y_UNUSED(status);
1496+
Y_UNUSED(error);
1497+
out.SetName(in.name());
1498+
if (in.has_min_value()) {
1499+
out.SetMinValue(in.min_value());
1500+
}
1501+
if (in.has_max_value()) {
1502+
out.SetMaxValue(in.max_value());
1503+
}
1504+
if (in.has_start_value()) {
1505+
out.SetStartValue(in.start_value());
1506+
}
1507+
if (in.has_cache()) {
1508+
out.SetCache(in.cache());
1509+
}
1510+
if (in.has_increment()) {
1511+
out.SetIncrement(in.increment());
1512+
}
1513+
if (in.has_cycle()) {
1514+
out.SetCycle(in.cycle());
1515+
}
1516+
if (in.has_set_val()) {
1517+
auto* setVal = out.MutableSetVal();
1518+
setVal->SetNextUsed(in.set_val().next_used());
1519+
setVal->SetNextValue(in.set_val().next_value());
1520+
}
1521+
return true;
14811522
}
14821523

14831524
} // namespace NKikimr

‎ydb/core/ydb_convert/table_description.h

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -133,11 +133,13 @@ bool FillTableDescription(NKikimrSchemeOp::TModifyScheme& out,
133133

134134

135135
// out
136-
void FillSequenceDescription(Ydb::Table::CreateTableRequest& out,
137-
const NKikimrSchemeOp::TTableDescription& in);
136+
bool FillSequenceDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error);
138137

139-
// out
140-
void FillSequenceDescription(Ydb::Table::CreateTableRequest& out,
141-
const NKikimrSchemeOp::TTableDescription& in);
138+
bool FillSequenceDescription(Ydb::Table::CreateTableRequest& out, const NKikimrSchemeOp::TTableDescription& in, Ydb::StatusIds::StatusCode& status, TString& error);
139+
140+
// in
141+
bool FillSequenceDescription(
142+
NKikimrSchemeOp::TSequenceDescription& out, const Ydb::Table::SequenceDescription& in,
143+
Ydb::StatusIds::StatusCode& status, TString& error);
142144

143145
} // namespace NKikimr

‎ydb/library/backup/backup.cpp

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,7 @@ NTable::TTableDescription DescribeTable(TDriver driver, const TString& fullTable
372372
NTable::TTableClient client(driver);
373373

374374
TStatus status = client.RetryOperationSync([fullTablePath, &desc](NTable::TSession session) {
375-
auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true);
375+
auto settings = NTable::TDescribeTableSettings().WithKeyShardBoundary(true).WithSetVal(true);
376376
auto result = session.DescribeTable(fullTablePath, settings).GetValueSync();
377377

378378
VerifyStatus(result);
@@ -489,6 +489,7 @@ void BackupTable(TDriver driver, const TString& dbPrefix, const TString& backupP
489489
<< " backupPrefix: " << backupPrefix << " path: " << path);
490490

491491
auto desc = DescribeTable(driver, JoinDatabasePath(schemaOnly ? dbPrefix : backupPrefix, path));
492+
492493
auto proto = ProtoFromTableDescription(desc, preservePoolKinds);
493494

494495
TString schemaStr;
@@ -718,19 +719,26 @@ void BackupFolder(TDriver driver, const TString& database, const TString& relDbP
718719
// Restore
719720
////////////////////////////////////////////////////////////////////////////////
720721

721-
TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTableBuilder *builder) {
722+
TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTableBuilder *builder, std::optional<NTable::TSequenceDescription> sequenceDescription) {
722723
TStringStream ss;
723724
ss << "name: " << name << "; ";
724725
if (parser.GetKind() == TTypeParser::ETypeKind::Optional) {
725726
ss << " optional; ";
726727
parser.OpenOptional();
727728
}
729+
if (sequenceDescription.has_value()) {
730+
ss << "serial; ";
731+
}
728732
ss << "kind: " << parser.GetKind() << "; ";
729733
switch (parser.GetKind()) {
730734
case TTypeParser::ETypeKind::Primitive:
731735
ss << " type_id: " << parser.GetPrimitive() << "; ";
732736
if (builder) {
733-
builder->AddNullableColumn(name, parser.GetPrimitive());
737+
if (sequenceDescription.has_value()) {
738+
builder->AddSerialColumn(name, parser.GetPrimitive(), std::move(*sequenceDescription));
739+
} else {
740+
builder->AddNullableColumn(name, parser.GetPrimitive());
741+
}
734742
}
735743
break;
736744
case TTypeParser::ETypeKind::Decimal:
@@ -751,8 +759,19 @@ TString ProcessColumnType(const TString& name, TTypeParser parser, NTable::TTabl
751759
NTable::TTableDescription TableDescriptionFromProto(const Ydb::Table::CreateTableRequest& proto) {
752760
NTable::TTableBuilder builder;
753761

762+
std::optional<NTable::TSequenceDescription> sequenceDescription;
754763
for (const auto &col : proto.Getcolumns()) {
755-
LOG_DEBUG("AddNullableColumn: " << ProcessColumnType(col.Getname(), TType(col.Gettype()), &builder));
764+
if (col.from_sequence().name() == "_serial_column_" + col.name()) {
765+
NTable::TSequenceDescription currentSequenceDescription;
766+
if (col.from_sequence().has_set_val()) {
767+
NTable::TSequenceDescription::TSetVal setVal;
768+
setVal.NextUsed = col.from_sequence().set_val().next_used();
769+
setVal.NextValue = col.from_sequence().set_val().next_value();
770+
currentSequenceDescription.SetVal = std::move(setVal);
771+
}
772+
sequenceDescription = std::move(currentSequenceDescription);
773+
}
774+
LOG_DEBUG("AddColumn: " << ProcessColumnType(col.Getname(), TType(col.Gettype()), &builder, std::move(sequenceDescription)));
756775
}
757776

758777
for (const auto &primary : proto.Getprimary_key()) {
@@ -781,7 +800,7 @@ TString SerializeColumnsToString(const TVector<TColumn>& columns, TVector<TStrin
781800
if (BinarySearch(primary.cbegin(), primary.cend(), col.Name)) {
782801
ss << "primary; ";
783802
}
784-
ss << ProcessColumnType(col.Name, col.Type, nullptr) << Endl;
803+
ss << ProcessColumnType(col.Name, col.Type, nullptr, std::nullopt) << Endl;
785804
}
786805
// Cerr << "Parse column to : " << ss.Str() << Endl;
787806
return ss.Str();

‎ydb/public/api/protos/ydb_table.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,8 @@ message DescribeTableRequest {
737737
bool include_table_stats = 6;
738738
// Includes partition statistics (required include_table_statistics)
739739
bool include_partition_stats = 7;
740+
// Includes set_val settings for sequences
741+
bool include_set_val = 8;
740742
}
741743

742744
message DescribeTableResponse {

‎ydb/public/sdk/cpp/client/ydb_table/impl/table_client.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,10 @@ TAsyncDescribeTableResult TTableClient::TImpl::DescribeTable(const TString& sess
525525
request.set_include_partition_stats(true);
526526
}
527527

528+
if (settings.WithSetVal_) {
529+
request.set_include_set_val(true);
530+
}
531+
528532
auto promise = NewPromise<TDescribeTableResult>();
529533

530534
auto extractor = [promise, settings]

‎ydb/public/sdk/cpp/client/ydb_table/table.cpp

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,24 @@ class TTableDescription::TImpl {
290290
if (col.has_not_null()) {
291291
not_null = col.not_null();
292292
}
293-
Columns_.emplace_back(col.name(), col.type(), col.family(), not_null);
293+
std::optional<TSequenceDescription> sequenceDescription;
294+
switch (col.default_value_case()) {
295+
case Ydb::Table::ColumnMeta::kFromSequence: {
296+
if (col.from_sequence().name() == "_serial_column_" + col.name()) {
297+
TSequenceDescription currentSequenceDescription;
298+
if (col.from_sequence().has_set_val()) {
299+
TSequenceDescription::TSetVal setVal;
300+
setVal.NextUsed = col.from_sequence().set_val().next_used();
301+
setVal.NextValue = col.from_sequence().set_val().next_value();
302+
currentSequenceDescription.SetVal = std::move(setVal);
303+
}
304+
sequenceDescription = std::move(currentSequenceDescription);
305+
}
306+
break;
307+
}
308+
default: break;
309+
}
310+
Columns_.emplace_back(col.name(), col.type(), col.family(), not_null, std::move(sequenceDescription));
294311
}
295312

296313
// indexes
@@ -452,8 +469,8 @@ class TTableDescription::TImpl {
452469
return Proto_;
453470
}
454471

455-
void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull) {
456-
Columns_.emplace_back(name, type, family, notNull);
472+
void AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull, std::optional<TSequenceDescription> sequenceDescription) {
473+
Columns_.emplace_back(name, type, family, notNull, std::move(sequenceDescription));
457474
}
458475

459476
void SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns) {
@@ -724,8 +741,8 @@ const TVector<TKeyRange>& TTableDescription::GetKeyRanges() const {
724741
return Impl_->GetKeyRanges();
725742
}
726743

727-
void TTableDescription::AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull) {
728-
Impl_->AddColumn(name, type, family, notNull);
744+
void TTableDescription::AddColumn(const TString& name, const Ydb::Type& type, const TString& family, std::optional<bool> notNull, std::optional<TSequenceDescription> sequenceDescription) {
745+
Impl_->AddColumn(name, type, family, notNull, std::move(sequenceDescription));
729746
}
730747

731748
void TTableDescription::SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns) {
@@ -889,6 +906,15 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con
889906
if (column.NotNull.has_value()) {
890907
protoColumn.set_not_null(column.NotNull.value());
891908
}
909+
if (column.SequenceDescription.has_value()) {
910+
auto* fromSequence = protoColumn.mutable_from_sequence();
911+
if (column.SequenceDescription->SetVal.has_value()) {
912+
auto* setVal = fromSequence->mutable_set_val();
913+
setVal->set_next_value(column.SequenceDescription->SetVal->NextValue);
914+
setVal->set_next_used(column.SequenceDescription->SetVal->NextUsed);
915+
}
916+
fromSequence->set_name("_serial_column_" + column.Name);
917+
}
892918
}
893919

894920
for (const auto& pk : Impl_->GetPrimaryKeyColumns()) {
@@ -1096,7 +1122,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const EPrim
10961122
.EndOptional()
10971123
.Build();
10981124

1099-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false);
1125+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt);
11001126
return *this;
11011127
}
11021128

@@ -1106,7 +1132,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TDeci
11061132
.Decimal(type)
11071133
.EndOptional()
11081134
.Build();
1109-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false);
1135+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt);
11101136
return *this;
11111137
}
11121138

@@ -1115,7 +1141,7 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TPgTy
11151141
.Pg(type)
11161142
.Build();
11171143

1118-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false);
1144+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, false, std::nullopt);
11191145
return *this;
11201146
}
11211147

@@ -1124,7 +1150,7 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const EP
11241150
.Primitive(type)
11251151
.Build();
11261152

1127-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true);
1153+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt);
11281154
return *this;
11291155
}
11301156

@@ -1133,7 +1159,7 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TD
11331159
.Decimal(type)
11341160
.Build();
11351161

1136-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true);
1162+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt);
11371163
return *this;
11381164
}
11391165

@@ -1142,7 +1168,16 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TP
11421168
.Pg(type)
11431169
.Build();
11441170

1145-
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true);
1171+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::nullopt);
1172+
return *this;
1173+
}
1174+
1175+
TTableBuilder& TTableBuilder::AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family) {
1176+
auto columnType = TTypeBuilder()
1177+
.Primitive(type)
1178+
.Build();
1179+
1180+
TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family, true, std::move(sequenceDescription));
11461181
return *this;
11471182
}
11481183

0 commit comments

Comments
 (0)