Skip to content

Commit 75f5ca4

Browse files
san-kir-kspaikus
san-kir-k
and
spaikus
authored
Tuple layout pack/unpack (#4905)
Co-authored-by: Vyacheslav Boev <[email protected]>
1 parent 5ccb702 commit 75f5ca4

File tree

5 files changed

+1717
-255
lines changed

5 files changed

+1717
-255
lines changed

‎ydb/library/yql/minikql/comp_nodes/packed_tuple/packed_tuple_ut.cpp

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,114 @@ Y_UNIT_TEST(Pack) {
179179

180180
}
181181

182+
Y_UNIT_TEST(Unpack) {
183+
184+
TScopedAlloc alloc(__LOCATION__);
185+
186+
TColumnDesc kc1, kc2, pc1, pc2;
187+
188+
kc1.Role = EColumnRole::Key;
189+
kc1.DataSize = 8;
190+
191+
kc2.Role = EColumnRole::Key;
192+
kc2.DataSize = 4;
193+
194+
pc1.Role = EColumnRole::Payload;
195+
pc1.DataSize = 8;
196+
197+
pc2.Role = EColumnRole::Payload;
198+
pc2.DataSize = 4;
199+
200+
std::vector<TColumnDesc> columns{kc1, kc2, pc1, pc2};
201+
202+
auto tl = TTupleLayout::Create(columns);
203+
UNIT_ASSERT(tl->TotalRowSize == 29);
204+
205+
const ui64 NTuples1 = 10e6;
206+
207+
const ui64 Tuples1DataBytes = (tl->TotalRowSize) * NTuples1;
208+
209+
std::vector<ui64> col1(NTuples1, 0);
210+
std::vector<ui32> col2(NTuples1, 0);
211+
std::vector<ui64> col3(NTuples1, 0);
212+
std::vector<ui32> col4(NTuples1, 0);
213+
214+
std::vector<ui8> res(Tuples1DataBytes + 64, 0);
215+
216+
for (ui32 i = 0; i < NTuples1; ++i) {
217+
col1[i] = i;
218+
col2[i] = i;
219+
col3[i] = i;
220+
col4[i] = i;
221+
}
222+
223+
const ui8* cols[4];
224+
225+
cols[0] = (ui8*) col1.data();
226+
cols[1] = (ui8*) col2.data();
227+
cols[2] = (ui8*) col3.data();
228+
cols[3] = (ui8*) col4.data();
229+
230+
std::vector<ui8> colValid1((NTuples1 + 7)/8, ~0);
231+
std::vector<ui8> colValid2((NTuples1 + 7)/8, ~0);
232+
std::vector<ui8> colValid3((NTuples1 + 7)/8, ~0);
233+
std::vector<ui8> colValid4((NTuples1 + 7)/8, ~0);
234+
const ui8 *colsValid[4] = {
235+
colValid1.data(),
236+
colValid2.data(),
237+
colValid3.data(),
238+
colValid4.data(),
239+
};
240+
241+
std::vector<ui8, TMKQLAllocator<ui8>> overflow;
242+
tl->Pack(cols, colsValid, res.data(), overflow, 0, NTuples1);
243+
244+
std::vector<ui64> col1_new(NTuples1, 0);
245+
std::vector<ui32> col2_new(NTuples1, 0);
246+
std::vector<ui64> col3_new(NTuples1, 0);
247+
std::vector<ui32> col4_new(NTuples1, 0);
248+
249+
ui8* cols_new[4];
250+
cols_new[0] = (ui8*) col1_new.data();
251+
cols_new[1] = (ui8*) col2_new.data();
252+
cols_new[2] = (ui8*) col3_new.data();
253+
cols_new[3] = (ui8*) col4_new.data();
254+
255+
std::vector<ui8> colValid1_new((NTuples1 + 7)/8, 0);
256+
std::vector<ui8> colValid2_new((NTuples1 + 7)/8, 0);
257+
std::vector<ui8> colValid3_new((NTuples1 + 7)/8, 0);
258+
std::vector<ui8> colValid4_new((NTuples1 + 7)/8, 0);
259+
260+
ui8 *colsValid_new[4] = {
261+
colValid1_new.data(),
262+
colValid2_new.data(),
263+
colValid3_new.data(),
264+
colValid4_new.data(),
265+
};
266+
267+
std::chrono::steady_clock::time_point begin02 = std::chrono::steady_clock::now();
268+
tl->Unpack(cols_new, colsValid_new, res.data(), overflow, 0, NTuples1);
269+
std::chrono::steady_clock::time_point end02 = std::chrono::steady_clock::now();
270+
ui64 microseconds = std::chrono::duration_cast<std::chrono::microseconds>(end02 - begin02).count();
271+
272+
if (microseconds == 0) microseconds = 1;
273+
274+
CTEST << "Time for " << (NTuples1) << " transpose (external cycle)= " << microseconds << "[microseconds]" << Endl;
275+
CTEST << "Data size = " << Tuples1DataBytes / (1024 * 1024) << "[MB]" << Endl;
276+
CTEST << "Calculating speed = " << Tuples1DataBytes / microseconds << "MB/sec" << Endl;
277+
CTEST << Endl;
278+
279+
UNIT_ASSERT(std::memcmp(col1.data(), col1_new.data(), sizeof(ui64) * col1.size()) == 0);
280+
UNIT_ASSERT(std::memcmp(col2.data(), col2_new.data(), sizeof(ui32) * col2.size()) == 0);
281+
UNIT_ASSERT(std::memcmp(col3.data(), col3_new.data(), sizeof(ui64) * col3.size()) == 0);
282+
UNIT_ASSERT(std::memcmp(col4.data(), col4_new.data(), sizeof(ui32) * col4.size()) == 0);
283+
284+
UNIT_ASSERT(std::memcmp(colValid1.data(), colValid1_new.data(), colValid1.size()) == 0);
285+
UNIT_ASSERT(std::memcmp(colValid2.data(), colValid2_new.data(), colValid2.size()) == 0);
286+
UNIT_ASSERT(std::memcmp(colValid3.data(), colValid3_new.data(), colValid3.size()) == 0);
287+
UNIT_ASSERT(std::memcmp(colValid4.data(), colValid4_new.data(), colValid4.size()) == 0);
288+
}
289+
182290
Y_UNIT_TEST(PackVarSize) {
183291

184292
TScopedAlloc alloc(__LOCATION__);
@@ -349,6 +457,197 @@ Y_UNIT_TEST(PackVarSize) {
349457
UNIT_ASSERT_VALUES_EQUAL(expected_overflow[i], overflow[i]);
350458
}
351459

460+
Y_UNIT_TEST(UnpackVarSize) {
461+
462+
TScopedAlloc alloc(__LOCATION__);
463+
464+
TColumnDesc kc1, kcv1, kcv2, kc2, pc1, pc2;
465+
466+
kc1.Role = EColumnRole::Key;
467+
kc1.DataSize = 8;
468+
469+
kc2.Role = EColumnRole::Key;
470+
kc2.DataSize = 4;
471+
472+
pc1.Role = EColumnRole::Payload;
473+
pc1.DataSize = 8;
474+
475+
pc2.Role = EColumnRole::Payload;
476+
pc2.DataSize = 4;
477+
478+
kcv1.Role = EColumnRole::Key;
479+
kcv1.DataSize = 8;
480+
kcv1.SizeType = EColumnSizeType::Variable;
481+
482+
kcv2.Role = EColumnRole::Key;
483+
kcv2.DataSize = 16;
484+
kcv2.SizeType = EColumnSizeType::Variable;
485+
486+
pc1.Role = EColumnRole::Payload;
487+
pc1.DataSize = 8;
488+
489+
pc2.Role = EColumnRole::Payload;
490+
pc2.DataSize = 4;
491+
492+
std::vector<TColumnDesc> columns{kc1, kc2, kcv1, kcv2, pc1, pc2};
493+
494+
auto tl = TTupleLayout::Create(columns);
495+
CTEST << "TotalRowSize = " << tl->TotalRowSize << Endl;
496+
UNIT_ASSERT_VALUES_EQUAL(tl->TotalRowSize, 54);
497+
498+
const ui64 NTuples1 = 3;
499+
500+
const ui64 Tuples1DataBytes = (tl->TotalRowSize) * NTuples1;
501+
502+
std::vector<ui64> col1(NTuples1, 0);
503+
std::vector<ui32> col2(NTuples1, 0);
504+
std::vector<ui64> col3(NTuples1, 0);
505+
std::vector<ui32> col4(NTuples1, 0);
506+
507+
std::vector<ui32> vcol1(1, 0);
508+
std::vector<ui8> vcol1data;
509+
std::vector<ui32> vcol2(1, 0);
510+
std::vector<ui8> vcol2data;
511+
512+
std::vector<ui8> res(Tuples1DataBytes + 64, 0);
513+
std::vector<TString> vcol1str {
514+
"abc",
515+
"ABCDEFGHIJKLMNO",
516+
"ZYXWVUTSPR"
517+
};
518+
std::vector<TString> vcol2str {
519+
"ABC",
520+
"abcdefghijklmno",
521+
"zyxwvutspr"
522+
};
523+
for (auto &&str: vcol1str) {
524+
for (auto c: str)
525+
vcol1data.push_back(c);
526+
vcol1.push_back(vcol1data.size());
527+
}
528+
UNIT_ASSERT_VALUES_EQUAL(vcol1.size(), NTuples1 + 1);
529+
for (auto &&str: vcol2str) {
530+
for (auto c: str)
531+
vcol2data.push_back(c);
532+
vcol2.push_back(vcol2data.size());
533+
}
534+
UNIT_ASSERT_VALUES_EQUAL(vcol2.size(), NTuples1 + 1);
535+
for (ui32 i = 0; i < NTuples1; ++i) {
536+
col1[i] = (1ull<<(sizeof(col1[0])*8 - 4)) + i + 1;
537+
col2[i] = (2ull<<(sizeof(col2[0])*8 - 4)) + i + 1;
538+
col3[i] = (3ull<<(sizeof(col3[0])*8 - 4)) + i + 1;
539+
col4[i] = (4ull<<(sizeof(col4[0])*8 - 4)) + i + 1;
540+
}
541+
542+
const ui8* cols[4 + 2*2];
543+
544+
cols[0] = (ui8*) col1.data();
545+
cols[1] = (ui8*) col2.data();
546+
cols[2] = (ui8*) vcol1.data();
547+
cols[3] = (ui8*) vcol1data.data();
548+
cols[4] = (ui8*) vcol2.data();
549+
cols[5] = (ui8*) vcol2data.data();
550+
cols[6] = (ui8*) col3.data();
551+
cols[7] = (ui8*) col4.data();
552+
553+
std::vector<ui8, TMKQLAllocator<ui8>> overflow;
554+
std::vector<ui8> colValid((NTuples1 + 7)/8, ~0);
555+
const ui8 *colsValid[8] = {
556+
colValid.data(),
557+
colValid.data(),
558+
colValid.data(),
559+
nullptr,
560+
colValid.data(),
561+
nullptr,
562+
colValid.data(),
563+
colValid.data(),
564+
};
565+
566+
tl->Pack(cols, colsValid, res.data(), overflow, 0, NTuples1);
567+
568+
std::vector<ui64> col1_new(NTuples1, 0);
569+
std::vector<ui32> col2_new(NTuples1, 0);
570+
std::vector<ui64> col3_new(NTuples1, 0);
571+
std::vector<ui32> col4_new(NTuples1, 0);
572+
573+
std::vector<ui32> vcol1_new(NTuples1 + 1, 0);
574+
std::vector<ui8> vcol1data_new(vcol1data.size());
575+
std::vector<ui32> vcol2_new(NTuples1 + 1, 0);
576+
std::vector<ui8> vcol2data_new(vcol2data.size());
577+
578+
ui8* cols_new[4 + 2 * 2];
579+
cols_new[0] = (ui8*) col1_new.data();
580+
cols_new[1] = (ui8*) col2_new.data();
581+
cols_new[2] = (ui8*) vcol1_new.data();
582+
cols_new[3] = (ui8*) vcol1data_new.data();
583+
cols_new[4] = (ui8*) vcol2_new.data();
584+
cols_new[5] = (ui8*) vcol2data_new.data();
585+
cols_new[6] = (ui8*) col3_new.data();
586+
cols_new[7] = (ui8*) col4_new.data();
587+
588+
std::vector<ui8> colValid1_new((NTuples1 + 7)/8, 0);
589+
colValid1_new.back() = ~0;
590+
std::vector<ui8> colValid2_new((NTuples1 + 7)/8, 0);
591+
colValid2_new.back() = ~0;
592+
std::vector<ui8> colValid3_new((NTuples1 + 7)/8, 0);
593+
colValid3_new.back() = ~0;
594+
std::vector<ui8> colValid4_new((NTuples1 + 7)/8, 0);
595+
colValid4_new.back() = ~0;
596+
std::vector<ui8> colValid5_new((NTuples1 + 7)/8, 0);
597+
colValid5_new.back() = ~0;
598+
std::vector<ui8> colValid6_new((NTuples1 + 7)/8, 0);
599+
colValid6_new.back() = ~0;
600+
601+
ui8 *colsValid_new[8] = {
602+
colValid1_new.data(),
603+
colValid2_new.data(),
604+
colValid3_new.data(),
605+
nullptr,
606+
colValid4_new.data(),
607+
nullptr,
608+
colValid5_new.data(),
609+
colValid6_new.data(),
610+
};
611+
612+
std::chrono::steady_clock::time_point begin02 = std::chrono::steady_clock::now();
613+
tl->Unpack(cols_new, colsValid_new, res.data(), overflow, 0, NTuples1);
614+
std::chrono::steady_clock::time_point end02 = std::chrono::steady_clock::now();
615+
ui64 microseconds = std::chrono::duration_cast<std::chrono::microseconds>(end02 - begin02).count();
616+
617+
if (microseconds == 0)
618+
microseconds = 1;
619+
620+
CTEST << "Time for " << (NTuples1) << " transpose (external cycle)= " << microseconds << "[microseconds]" << Endl;
621+
#ifndef NDEBUG
622+
CTEST << "Result size = " << Tuples1DataBytes << Endl;
623+
CTEST << "Result = ";
624+
for (ui32 i = 0; i < Tuples1DataBytes; ++i)
625+
CTEST << int(res[i]) << ' ';
626+
CTEST << Endl;
627+
CTEST << "Overflow size = " << overflow.size() << Endl;
628+
CTEST << "Overflow = ";
629+
for (auto c: overflow)
630+
CTEST << int(c) << ' ';
631+
CTEST << Endl;
632+
#endif
633+
634+
UNIT_ASSERT(std::memcmp(cols[0], cols_new[0], sizeof(ui64) * col1.size()) == 0);
635+
UNIT_ASSERT(std::memcmp(cols[1], cols_new[1], sizeof(ui32) * col2.size()) == 0);
636+
UNIT_ASSERT(std::memcmp(cols[2], cols_new[2], sizeof(ui32) * vcol1.size()) == 0);
637+
UNIT_ASSERT(std::memcmp(cols[3], cols_new[3], vcol1data.size()) == 0);
638+
UNIT_ASSERT(std::memcmp(cols[4], cols_new[4], sizeof(ui32) * vcol2.size()) == 0);
639+
UNIT_ASSERT(std::memcmp(cols[5], cols_new[5], vcol1data.size()) == 0);
640+
UNIT_ASSERT(std::memcmp(cols[6], cols_new[6], sizeof(ui64) * col3.size()) == 0);
641+
UNIT_ASSERT(std::memcmp(cols[7], cols_new[7], sizeof(ui32) * col4.size()) == 0);
642+
643+
UNIT_ASSERT(std::memcmp(colValid.data(), colValid1_new.data(), colValid.size()) == 0);
644+
UNIT_ASSERT(std::memcmp(colValid.data(), colValid2_new.data(), colValid.size()) == 0);
645+
UNIT_ASSERT(std::memcmp(colValid.data(), colValid3_new.data(), colValid.size()) == 0);
646+
UNIT_ASSERT(std::memcmp(colValid.data(), colValid4_new.data(), colValid.size()) == 0);
647+
UNIT_ASSERT(std::memcmp(colValid.data(), colValid5_new.data(), colValid.size()) == 0);
648+
UNIT_ASSERT(std::memcmp(colValid.data(), colValid6_new.data(), colValid.size()) == 0);
649+
}
650+
352651
Y_UNIT_TEST(PackVarSizeBig) {
353652

354653
TScopedAlloc alloc(__LOCATION__);

0 commit comments

Comments
 (0)