Skip to content

Commit 0abd28e

Browse files
dcherednikSammyVimes
dcherednik
and
SammyVimes
authored
Merge UUID related fixes for export (#6697)
Co-authored-by: Semyon Danilov <[email protected]>
1 parent 5ec8b8c commit 0abd28e

File tree

11 files changed

+248
-1
lines changed

11 files changed

+248
-1
lines changed

‎ydb/core/io_formats/cell_maker/cell_maker.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include <ydb/library/binary_json/write.h>
44
#include <ydb/library/dynumber/dynumber.h>
5+
#include <ydb/library/uuid/uuid.h>
56

67
#include <ydb/library/yql/minikql/dom/yson.h>
78
#include <ydb/library/yql/minikql/dom/json.h>
@@ -132,6 +133,21 @@ namespace {
132133
return true;
133134
}
134135

136+
struct TUuidHolder {
137+
union {
138+
ui16 Array[8];
139+
ui64 Halves[2];
140+
} Buf;
141+
};
142+
143+
template <>
144+
bool TryParse(TStringBuf value, TUuidHolder& result) {
145+
if (!NUuid::ParseUuidToArray(value, result.Buf.Array, false)) {
146+
return false;
147+
}
148+
return true;
149+
}
150+
135151
template <typename T, typename U>
136152
using TConverter = std::function<U(const T&)>;
137153

@@ -171,6 +187,14 @@ namespace {
171187
return v.Str;
172188
}
173189

190+
TStringBuf UuidToStringBuf(const TUuidHolder& uuid) {
191+
char uuidBuf[16];
192+
193+
NUuid::UuidHalfsToBytes(uuidBuf, 16, uuid.Buf.Halves[1], uuid.Buf.Halves[0]);
194+
195+
return TStringBuf(uuidBuf, 16);
196+
}
197+
174198
template <typename T, typename U = T>
175199
struct TCellMaker {
176200
static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TString& err, TConverter<T, U> conv = &Implicit<T, U>) {
@@ -297,6 +321,8 @@ bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeInfo type, TMemoryPoo
297321
return TCellMaker<NYql::NDecimal::TInt128, std::pair<ui64, ui64>>::Make(cell, value, pool, err, &Int128ToPair);
298322
case NScheme::NTypeIds::Pg:
299323
return TCellMaker<NPg::TConvertResult, TStringBuf>::Make(cell, value, pool, err, &PgToStringBuf, type.GetTypeDesc());
324+
case NScheme::NTypeIds::Uuid:
325+
return TCellMaker<TUuidHolder, TStringBuf>::Make(cell, value, pool, err, &UuidToStringBuf);
300326
default:
301327
return false;
302328
}
@@ -390,6 +416,7 @@ bool CheckCellValue(const TCell& cell, NScheme::TTypeInfo type) {
390416
case NScheme::NTypeIds::JsonDocument: // checked at parsing time
391417
case NScheme::NTypeIds::DyNumber: // checked at parsing time
392418
case NScheme::NTypeIds::Pg: // checked at parsing time
419+
case NScheme::NTypeIds::Uuid: // checked at parsing time
393420
return true;
394421
case NScheme::NTypeIds::Date:
395422
return cell.AsValue<ui16>() < NUdf::MAX_DATE;

‎ydb/core/kqp/opt/kqp_query_plan.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,12 @@ class TxPlanSerializer {
560560
}
561561
}
562562

563+
if (auto literal = key.Maybe<TCoUuid>()) {
564+
TStringStream out;
565+
NUuid::UuidBytesToString(literal.Cast().Literal().Value().Data(), out);
566+
return out.Str();
567+
}
568+
563569
if (auto literal = key.Maybe<TCoDataCtor>()) {
564570
return literal.Cast().Literal().StringValue();
565571
}

‎ydb/core/kqp/provider/yql_kikimr_provider.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -605,6 +605,12 @@ void FillLiteralProtoImpl(const NNodes::TCoDataCtor& literal, TProto& proto) {
605605
protoValue.SetHi128(*reinterpret_cast<ui64*>(p + 8));
606606
break;
607607
}
608+
case EDataSlot::Uuid: {
609+
const ui64* uuidData = reinterpret_cast<const ui64*>(value.Data());
610+
protoValue.SetLow128(uuidData[0]);
611+
protoValue.SetHi128(uuidData[1]);
612+
break;
613+
}
608614

609615
default:
610616
YQL_ENSURE(false, "Unexpected type slot " << slot);
@@ -738,6 +744,12 @@ void FillLiteralProto(const NNodes::TCoDataCtor& literal, Ydb::TypedValue& proto
738744
protoValue.set_high_128(*reinterpret_cast<ui64*>(p + 8));
739745
break;
740746
}
747+
case EDataSlot::Uuid: {
748+
const ui64* uuidData = reinterpret_cast<const ui64*>(value.Data());
749+
protoValue.set_low_128(uuidData[0]);
750+
protoValue.set_high_128(uuidData[1]);
751+
break;
752+
}
741753

742754
default:
743755
YQL_ENSURE(false, "Unexpected type slot " << slot);

‎ydb/core/kqp/ut/yql/kqp_yql_ut.cpp

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,94 @@ Y_UNIT_TEST_SUITE(KqpYql) {
740740
}
741741
}
742742

743+
Y_UNIT_TEST(TestUuidPrimaryKeyPrefixSearch) {
744+
NKikimrConfig::TAppConfig appConfig;
745+
appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true);
746+
auto setting = NKikimrKqp::TKqpSetting();
747+
auto serverSettings = TKikimrSettings()
748+
.SetAppConfig(appConfig)
749+
.SetEnableUuidAsPrimaryKey(true)
750+
.SetKqpSettings({setting});
751+
TKikimrRunner kikimr(serverSettings.SetWithSampleTables(false));
752+
753+
auto db = kikimr.GetTableClient();
754+
auto session = db.CreateSession().GetValueSync().GetSession();
755+
756+
TVector<TString> testUuids = {
757+
"5b99a330-04ef-4f1a-9b64-ba6d5f44eafe",
758+
"afcbef30-9ac3-481a-aa6a-8d9b785dbb0a",
759+
"b91cd23b-861c-4cc1-9119-801a4dac1cb9",
760+
"65df9ecc-a97d-47b2-ae56-3c023da6ee8c",
761+
};
762+
763+
{
764+
const auto query = Q_(R"(
765+
CREATE TABLE test(
766+
key uuid NOT NULL,
767+
val int,
768+
PRIMARY KEY (key)
769+
);
770+
)");
771+
auto result = session.ExecuteSchemeQuery(query).ExtractValueSync();
772+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
773+
}
774+
{
775+
int val = 0;
776+
for (const auto& uuid : testUuids) {
777+
const auto query = Sprintf("\
778+
INSERT INTO test (key, val)\n\
779+
VALUES (Uuid(\"%s\"), %u);\n\
780+
", uuid.Data(), val++);
781+
auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
782+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
783+
}
784+
}
785+
{
786+
int val = 0;
787+
for (const auto& uuid : testUuids) {
788+
const auto query = Sprintf("SELECT * FROM test WHERE key=Uuid(\"%s\");", uuid.Data());
789+
auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
790+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
791+
792+
TResultSetParser parser(result.GetResultSetParser(0));
793+
UNIT_ASSERT(parser.TryNextRow());
794+
UNIT_ASSERT_VALUES_EQUAL(parser.ColumnParser("val").GetOptionalInt32().GetRef(), val++);
795+
UNIT_ASSERT_VALUES_EQUAL(parser.RowsCount(), 1);
796+
}
797+
}
798+
}
799+
800+
Y_UNIT_TEST(TestUuidDefaultColumn) {
801+
NKikimrConfig::TAppConfig appConfig;
802+
appConfig.MutableTableServiceConfig()->SetEnablePreparedDdl(true);
803+
auto setting = NKikimrKqp::TKqpSetting();
804+
auto serverSettings = TKikimrSettings()
805+
.SetAppConfig(appConfig)
806+
.SetEnableUuidAsPrimaryKey(true)
807+
.SetKqpSettings({setting});
808+
TKikimrRunner kikimr(serverSettings.SetWithSampleTables(false));
809+
810+
auto db = kikimr.GetTableClient();
811+
auto session = db.CreateSession().GetValueSync().GetSession();
812+
813+
{
814+
const auto query = Q_(R"(
815+
CREATE TABLE test(
816+
key int NOT NULL,
817+
val uuid NOT NULL DEFAULT Uuid("65df9ecc-a97d-47b2-ae56-3c023da6ee8c"),
818+
PRIMARY KEY (key)
819+
);
820+
)");
821+
auto result = session.ExecuteSchemeQuery(query).ExtractValueSync();
822+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
823+
}
824+
{
825+
const auto query = "INSERT INTO test (key) VALUES (0);";
826+
auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync();
827+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
828+
}
829+
}
830+
743831
Y_UNIT_TEST(UuidPrimaryKeyBulkUpsert) {
744832
auto settings = TKikimrSettings()
745833
.SetEnableUuidAsPrimaryKey(true)

‎ydb/core/tx/datashard/export_common.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,5 +118,15 @@ bool PgToStream(TStringBuf data, void* typeDesc, IOutputStream& out, TString& er
118118
return true;
119119
}
120120

121+
bool UuidToStream(const std::pair<ui64, ui64>& loHi, IOutputStream& out, TString& err) {
122+
Y_UNUSED(err);
123+
124+
NYdb::TUuidValue uuid(loHi.first, loHi.second);
125+
126+
out << uuid.ToString();
127+
128+
return true;
129+
}
130+
121131
} // NDataShard
122132
} // NKikimr

‎ydb/core/tx/datashard/export_common.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ TString DyNumberToString(TStringBuf data);
4040
bool DecimalToStream(const std::pair<ui64, i64>& loHi, IOutputStream& out, TString& err);
4141
bool DyNumberToStream(TStringBuf data, IOutputStream& out, TString& err);
4242
bool PgToStream(TStringBuf data, void* typeDesc, IOutputStream& out, TString& err);
43+
bool UuidToStream(const std::pair<ui64, ui64>& loHi, IOutputStream& out, TString& err);
4344

4445
} // NDataShard
4546
} // NKikimr

‎ydb/core/tx/datashard/export_s3_buffer_raw.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,9 @@ bool TS3BufferRaw::Collect(const NTable::IScan::TRow& row, IOutputStream& out) {
125125
case NScheme::NTypeIds::Pg:
126126
serialized = PgToStream(cell.AsBuf(), column.Type.GetTypeDesc(), out, ErrorString);
127127
break;
128+
case NScheme::NTypeIds::Uuid:
129+
serialized = UuidToStream(cell.AsValue<std::pair<ui64, ui64>>(), out, ErrorString);
130+
break;
128131
default:
129132
Y_ABORT("Unsupported type");
130133
}

‎ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,30 @@ Y_UNIT_TEST_SUITE(TBackupTests) {
100100
});
101101
}
102102

103+
Y_UNIT_TEST_WITH_COMPRESSION(BackupUuidColumn) {
104+
TTestBasicRuntime runtime;
105+
106+
Backup(runtime, ToString(Codec), R"(
107+
Name: "Table"
108+
Columns { Name: "key" Type: "Uint32" }
109+
Columns { Name: "value" Type: "Uuid" }
110+
KeyColumnNames: ["key"]
111+
)", [](TTestBasicRuntime& runtime) {
112+
NKikimrMiniKQL::TResult result;
113+
TString error;
114+
NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, TTestTxConfig::FakeHiveTablets, Sprintf(R"(
115+
(
116+
(let key '( '('key (Uint32 '%d) ) ) )
117+
(let row '( '('value (Uuid '"%s") ) ) )
118+
(return (AsList (UpdateRow '__user__%s key row) ))
119+
)
120+
)", 1, "0000111122223333", "Table"), result, error);
121+
122+
UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error);
123+
UNIT_ASSERT_VALUES_EQUAL(error, "");
124+
});
125+
}
126+
103127
template<ECompressionCodec Codec>
104128
void ShouldSucceedOnLargeData(ui32 minWriteBatchSize, const std::pair<ui32, ui32>& expectedResult) {
105129
TTestBasicRuntime runtime;

‎ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1135,7 +1135,76 @@ value {
11351135
env.TestWaitNotification(runtime, txId);
11361136
TestGetExport(runtime, txId, "/MyRoot");
11371137

1138-
TestImport(runtime, txId, "/MyRoot", Sprintf(R"(
1138+
TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
1139+
ImportFromS3Settings {
1140+
endpoint: "localhost:%d"
1141+
scheme: HTTP
1142+
items {
1143+
source_prefix: "Backup1"
1144+
destination_path: "/MyRoot/Restored"
1145+
}
1146+
}
1147+
)", port));
1148+
env.TestWaitNotification(runtime, txId);
1149+
TestGetImport(runtime, txId, "/MyRoot");
1150+
}
1151+
1152+
Y_UNIT_TEST(ExportImportUuid) {
1153+
TTestBasicRuntime runtime;
1154+
TTestEnv env(runtime, TTestEnvOptions().EnableTablePgTypes(true));
1155+
ui64 txId = 100;
1156+
1157+
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
1158+
Name: "Table"
1159+
Columns { Name: "key" Type: "Uint32" }
1160+
Columns { Name: "value" Type: "Uuid" }
1161+
KeyColumnNames: ["key"]
1162+
)");
1163+
env.TestWaitNotification(runtime, txId);
1164+
1165+
{
1166+
TString tablePath = "/MyRoot/Table";
1167+
int partitionIdx = 0;
1168+
1169+
auto tableDesc = DescribePath(runtime, tablePath, true, true);
1170+
const auto& tablePartitions = tableDesc.GetPathDescription().GetTablePartitions();
1171+
UNIT_ASSERT(partitionIdx < tablePartitions.size());
1172+
const ui64 datashardTabletId = tablePartitions[partitionIdx].GetDatashardId();
1173+
1174+
NKikimrMiniKQL::TResult result;
1175+
TString error;
1176+
NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, datashardTabletId, Sprintf(R"(
1177+
(
1178+
(let key '( '('key (Uint32 '%d) ) ) )
1179+
(let row '( '('value (Uuid '"%s") ) ) )
1180+
(return (AsList (UpdateRow '__user__%s key row) ))
1181+
)
1182+
)", 1, "0123456789012345", "Table"), result, error);
1183+
1184+
UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error);
1185+
UNIT_ASSERT_VALUES_EQUAL(error, "");
1186+
}
1187+
1188+
TPortManager portManager;
1189+
const ui16 port = portManager.GetPort();
1190+
1191+
TS3Mock s3Mock({}, TS3Mock::TSettings(port));
1192+
UNIT_ASSERT(s3Mock.Start());
1193+
1194+
TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"(
1195+
ExportToS3Settings {
1196+
endpoint: "localhost:%d"
1197+
scheme: HTTP
1198+
items {
1199+
source_path: "/MyRoot/Table"
1200+
destination_prefix: "Backup1"
1201+
}
1202+
}
1203+
)", port));
1204+
env.TestWaitNotification(runtime, txId);
1205+
TestGetExport(runtime, txId, "/MyRoot");
1206+
1207+
TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
11391208
ImportFromS3Settings {
11401209
endpoint: "localhost:%d"
11411210
scheme: HTTP

‎ydb/library/uuid/uuid.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@ static void WriteHex(ui16 bytes, IOutputStream& out, bool reverseBytes = false)
2828
}
2929
}
3030

31+
void UuidBytesToString(TString in, IOutputStream& out) {
32+
ui16 dw[8];
33+
std::memcpy(dw, in.Data(), sizeof(dw));
34+
NUuid::UuidToString(dw, out);
35+
}
36+
3137
void UuidToString(ui16 dw[8], IOutputStream& out) {
3238
WriteHex(dw[1], out);
3339
WriteHex(dw[0], out);

‎ydb/library/uuid/uuid.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ namespace NUuid {
1313

1414
static constexpr ui32 UUID_LEN = 16;
1515

16+
void UuidBytesToString(TString in, IOutputStream& out);
1617
void UuidToString(ui16 dw[8], IOutputStream& out);
1718
void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out);
1819

0 commit comments

Comments
 (0)