Skip to content

Commit 2d35372

Browse files
shnikdgridnevvvit
shnikd
authored and
gridnevvvit
committed
Add default policy to ColumnMeta (ydb-platform#1791)
1 parent 0fd6659 commit 2d35372

File tree

4 files changed

+275
-4
lines changed

4 files changed

+275
-4
lines changed

‎ydb/core/tx/schemeshard/ut_export/ut_export.cpp

+64-2
Original file line numberDiff line numberDiff line change
@@ -281,8 +281,42 @@ Y_UNIT_TEST_SUITE(TExportToS3Tests) {
281281

282282
const TVector<TString> tables = {R"(
283283
Name: "Table"
284-
Columns { Name: "key" Type: "Utf8" }
285-
Columns { Name: "value" Type: "Utf8" }
284+
Columns {
285+
Name: "key"
286+
Type: "Utf8"
287+
DefaultFromLiteral {
288+
type {
289+
optional_type {
290+
item {
291+
type_id: UTF8
292+
}
293+
}
294+
}
295+
value {
296+
items {
297+
text_value: "b"
298+
}
299+
}
300+
}
301+
}
302+
Columns {
303+
Name: "value"
304+
Type: "Utf8"
305+
DefaultFromLiteral {
306+
type {
307+
optional_type {
308+
item {
309+
type_id: UTF8
310+
}
311+
}
312+
}
313+
value {
314+
items {
315+
text_value: "a"
316+
}
317+
}
318+
}
319+
}
286320
KeyColumnNames: ["key"]
287321
PartitionConfig {
288322
ColumnFamilies {
@@ -331,6 +365,20 @@ Y_UNIT_TEST_SUITE(TExportToS3Tests) {
331365
}
332366
}
333367
not_null: false
368+
from_literal {
369+
type {
370+
optional_type {
371+
item {
372+
type_id: UTF8
373+
}
374+
}
375+
}
376+
value {
377+
items {
378+
text_value: "b"
379+
}
380+
}
381+
}
334382
}
335383
columns {
336384
name: "value"
@@ -342,6 +390,20 @@ columns {
342390
}
343391
}
344392
not_null: false
393+
from_literal {
394+
type {
395+
optional_type {
396+
item {
397+
type_id: UTF8
398+
}
399+
}
400+
}
401+
value {
402+
items {
403+
text_value: "a"
404+
}
405+
}
406+
}
345407
}
346408
primary_key: "key"
347409
storage_settings {

‎ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp

+148
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,79 @@ Y_UNIT_TEST_SUITE(TRestoreTests) {
373373
NKqp::CompareYson(data.YsonStr, content);
374374
}
375375

376+
bool CheckDefaultFromLiteral(const NKikimrSchemeOp::TTableDescription& desc) {
377+
for (const auto& column: desc.GetColumns()) {
378+
if (column.GetName() == "value") {
379+
switch (column.GetDefaultValueCase()) {
380+
case NKikimrSchemeOp::TColumnDescription::kDefaultFromLiteral: {
381+
const auto& fromLiteral = column.GetDefaultFromLiteral();
382+
383+
TString str;
384+
google::protobuf::TextFormat::PrintToString(fromLiteral, &str);
385+
386+
TString result = R"(type {
387+
optional_type {
388+
item {
389+
type_id: UTF8
390+
}
391+
}
392+
}
393+
value {
394+
items {
395+
text_value: "value1"
396+
}
397+
}
398+
)";
399+
return str == result;
400+
}
401+
default: break;
402+
}
403+
break;
404+
}
405+
}
406+
return false;
407+
}
408+
409+
Y_UNIT_TEST_WITH_COMPRESSION(ShouldSucceedWithDefaultFromLiteral) {
410+
TTestBasicRuntime runtime;
411+
412+
const auto data = GenerateTestData(Codec, "a", 1);
413+
414+
Restore(runtime, R"(
415+
Name: "Table"
416+
Columns { Name: "key" Type: "Utf8" }
417+
Columns {
418+
Name: "value"
419+
Type: "Utf8"
420+
DefaultFromLiteral {
421+
type {
422+
optional_type {
423+
item {
424+
type_id: UTF8
425+
}
426+
}
427+
}
428+
value {
429+
items {
430+
text_value: "value1"
431+
}
432+
}
433+
}
434+
}
435+
KeyColumnNames: ["key"]
436+
)", {data});
437+
438+
auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets);
439+
NKqp::CompareYson(data.YsonStr, content);
440+
441+
const auto desc = DescribePath(runtime, "/MyRoot/Table", true, true);
442+
UNIT_ASSERT_VALUES_EQUAL(desc.GetStatus(), NKikimrScheme::StatusSuccess);
443+
444+
const auto& table = desc.GetPathDescription().GetTable();
445+
446+
UNIT_ASSERT_C(CheckDefaultFromLiteral(table), "Invalid default value");
447+
}
448+
376449
Y_UNIT_TEST_WITH_COMPRESSION(ShouldSucceedOnMultiShardTable) {
377450
TTestBasicRuntime runtime;
378451

@@ -729,6 +802,81 @@ Y_UNIT_TEST_SUITE(TRestoreTests) {
729802
TestGetImport(runtime, txId, "/MyRoot");
730803
}
731804

805+
Y_UNIT_TEST(ShouldRestoreDefaultValuesFromLiteral) {
806+
TPortManager portManager;
807+
const ui16 port = portManager.GetPort();
808+
809+
TS3Mock s3Mock({}, TS3Mock::TSettings(port));
810+
UNIT_ASSERT(s3Mock.Start());
811+
812+
TTestBasicRuntime runtime;
813+
TTestEnv env(runtime);
814+
ui64 txId = 100;
815+
816+
runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE);
817+
runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE);
818+
runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE);
819+
runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE);
820+
821+
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
822+
Name: "Original"
823+
Columns { Name: "key" Type: "Utf8" }
824+
Columns {
825+
Name: "value"
826+
Type: "Utf8"
827+
DefaultFromLiteral {
828+
type {
829+
optional_type {
830+
item {
831+
type_id: UTF8
832+
}
833+
}
834+
}
835+
value {
836+
items {
837+
text_value: "value1"
838+
}
839+
}
840+
}
841+
}
842+
KeyColumnNames: ["key"]
843+
)");
844+
env.TestWaitNotification(runtime, txId);
845+
846+
TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"(
847+
ExportToS3Settings {
848+
endpoint: "localhost:%d"
849+
scheme: HTTP
850+
items {
851+
source_path: "/MyRoot/Original"
852+
destination_prefix: ""
853+
}
854+
}
855+
)", port));
856+
env.TestWaitNotification(runtime, txId);
857+
TestGetExport(runtime, txId, "/MyRoot");
858+
859+
TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
860+
ImportFromS3Settings {
861+
endpoint: "localhost:%d"
862+
scheme: HTTP
863+
items {
864+
source_prefix: ""
865+
destination_path: "/MyRoot/Restored"
866+
}
867+
}
868+
)", port));
869+
env.TestWaitNotification(runtime, txId);
870+
TestGetImport(runtime, txId, "/MyRoot");
871+
872+
const auto desc = DescribePath(runtime, "/MyRoot/Restored", true, true);
873+
UNIT_ASSERT_VALUES_EQUAL(desc.GetStatus(), NKikimrScheme::StatusSuccess);
874+
875+
const auto& table = desc.GetPathDescription().GetTable();
876+
877+
UNIT_ASSERT_C(CheckDefaultFromLiteral(table), "Invalid default value");
878+
}
879+
732880
Y_UNIT_TEST(ExportImportPg) {
733881
TTestBasicRuntime runtime;
734882
TTestEnv env(runtime, TTestEnvOptions().EnableTablePgTypes(true));

‎ydb/core/ydb_convert/table_description.cpp

+59-2
Original file line numberDiff line numberDiff line change
@@ -151,11 +151,11 @@ bool BuildAlterTableAddIndexRequest(const Ydb::Table::AlterTableRequest* req, NK
151151
if (flags & NKqpProto::TKqpSchemeOperation::FLAG_PG_MODE) {
152152
settings->set_pg_mode(true);
153153
}
154-
154+
155155
if (flags & NKqpProto::TKqpSchemeOperation::FLAG_IF_NOT_EXISTS) {
156156
settings->set_if_not_exist(true);
157157
}
158-
158+
159159
settings->set_source_path(req->path());
160160
auto tableIndex = settings->mutable_index();
161161
tableIndex->CopyFrom(req->add_indexes(0));
@@ -390,6 +390,54 @@ static Ydb::Type* AddColumn(Ydb::Table::ColumnMeta* newColumn, const TColumn& co
390390
}
391391
}
392392
newColumn->set_not_null(column.GetNotNull());
393+
return columnType;
394+
}
395+
396+
template <>
397+
Ydb::Type* AddColumn<NKikimrSchemeOp::TColumnDescription>(Ydb::Table::ColumnMeta* newColumn, const NKikimrSchemeOp::TColumnDescription& column) {
398+
newColumn->set_name(column.GetName());
399+
400+
Ydb::Type* columnType = nullptr;
401+
auto* typeDesc = NPg::TypeDescFromPgTypeName(column.GetType());
402+
if (typeDesc) {
403+
columnType = newColumn->mutable_type();
404+
auto* pg = columnType->mutable_pg_type();
405+
pg->set_type_name(NPg::PgTypeNameFromTypeDesc(typeDesc));
406+
pg->set_type_modifier(NPg::TypeModFromPgTypeName(column.GetType()));
407+
pg->set_oid(NPg::PgTypeIdFromTypeDesc(typeDesc));
408+
pg->set_typlen(0);
409+
pg->set_typmod(0);
410+
} else {
411+
NYql::NProto::TypeIds protoType;
412+
if (!NYql::NProto::TypeIds_Parse(column.GetType(), &protoType)) {
413+
throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR)
414+
<< "Got invalid type: " << column.GetType() << " for column: " << column.GetName();
415+
}
416+
417+
if (column.GetNotNull()) {
418+
columnType = newColumn->mutable_type();
419+
} else {
420+
columnType = newColumn->mutable_type()->mutable_optional_type()->mutable_item();
421+
}
422+
Y_ENSURE(columnType);
423+
if (protoType == NYql::NProto::TypeIds::Decimal) {
424+
auto typeParams = columnType->mutable_decimal_type();
425+
// TODO: Change TEvDescribeSchemeResult to return decimal params
426+
typeParams->set_precision(22);
427+
typeParams->set_scale(9);
428+
} else {
429+
NMiniKQL::ExportPrimitiveTypeToProto(protoType, *columnType);
430+
}
431+
}
432+
newColumn->set_not_null(column.GetNotNull());
433+
switch (column.GetDefaultValueCase()) {
434+
case NKikimrSchemeOp::TColumnDescription::kDefaultFromLiteral: {
435+
auto fromLiteral = newColumn->mutable_from_literal();
436+
*fromLiteral = column.GetDefaultFromLiteral();
437+
break;
438+
}
439+
default: break;
440+
}
393441

394442
return columnType;
395443
}
@@ -607,6 +655,15 @@ bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out,
607655
if (!column.family().empty()) {
608656
cd->SetFamilyName(column.family());
609657
}
658+
659+
switch (column.default_value_case()) {
660+
case Ydb::Table::ColumnMeta::kFromLiteral: {
661+
auto fromLiteral = cd->MutableDefaultFromLiteral();
662+
*fromLiteral = column.from_literal();
663+
break;
664+
}
665+
default: break;
666+
}
610667
}
611668

612669
return true;

‎ydb/public/api/protos/ydb_table.proto

+4
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,10 @@ message ColumnMeta {
342342
string family = 3;
343343
// Column nullability
344344
optional bool not_null = 4;
345+
// Column default value option
346+
oneof default_value {
347+
TypedValue from_literal = 5;
348+
}
345349
}
346350

347351
message DateTypeColumnModeSettings {

0 commit comments

Comments
 (0)