Skip to content

Commit 1e82ca6

Browse files
lll-phill-llllll-phill-lll
committed
added more tests
1 parent bde88f9 commit 1e82ca6

File tree

3 files changed

+95
-26
lines changed

3 files changed

+95
-26
lines changed

‎ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
void N(NUnitTest::TTestContext&)
4949

5050
#define Y_UNIT_TEST_LLVM(N) Y_UNIT_TEST_TWIN(N, LLVM)
51+
#define Y_UNIT_TEST_LLVM_SPILLING(N) Y_UNIT_TEST_QUAD(N, LLVM, SPILLING)
5152

5253
#define Y_UNIT_TEST_QUAD(N, OPT1, OPT2) \
5354
template<bool OPT1, bool OPT2> void N(NUnitTest::TTestContext&); \
@@ -79,9 +80,9 @@ struct TUdfModuleInfo {
7980
NUdf::TUniquePtr<NUdf::IUdfModule> Module;
8081
};
8182

82-
template<bool UseLLVM>
83+
template<bool UseLLVM, bool EnableSpilling = false>
8384
struct TSetup {
84-
explicit TSetup(TComputationNodeFactory nodeFactory = GetTestFactory(), TVector<TUdfModuleInfo>&& modules = {}, bool withSpilling = false)
85+
explicit TSetup(TComputationNodeFactory nodeFactory = GetTestFactory(), TVector<TUdfModuleInfo>&& modules = {})
8586
: Alloc(__LOCATION__)
8687
, StatsRegistry(CreateDefaultStatsRegistry())
8788
{
@@ -96,7 +97,7 @@ struct TSetup {
9697
FunctionRegistry = mutableRegistry;
9798
}
9899

99-
Alloc.Ref().ForcefullySetMemoryYellowZone(withSpilling);
100+
Alloc.Ref().ForcefullySetMemoryYellowZone(EnableSpilling);
100101

101102
RandomProvider = CreateDeterministicRandomProvider(1);
102103
TimeProvider = CreateDeterministicTimeProvider(10000000);

‎ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp

Lines changed: 73 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1003,8 +1003,13 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideCombinerPerfTest) {
10031003
#endif
10041004
#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 29u
10051005
Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
1006-
Y_UNIT_TEST_LLVM(TestLongStringsRefCounting) {
1007-
TSetup<LLVM> setup(GetTestFactory(), {}, true);
1006+
Y_UNIT_TEST_LLVM_SPILLING(TestLongStringsRefCounting) {
1007+
// Currently LLVM version doesn't support spilling.
1008+
if (LLVM && SPILLING) return;
1009+
// callable WideLastCombinerWithSpilling was introduced in 49 version of runtime
1010+
if (MKQL_RUNTIME_VERSION < 49U && SPILLING) return;
1011+
1012+
TSetup<LLVM, SPILLING> setup;
10081013
TProgramBuilder& pb = *setup.PgmBuilder;
10091014

10101015
const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
@@ -1042,7 +1047,12 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
10421047

10431048
const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
10441049

1045-
const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombinerWithSpilling(pb.ExpandMap(pb.ToFlow(list),
1050+
auto wideLastCombinerCollable = &TProgramBuilder::WideLastCombiner;
1051+
if (SPILLING) {
1052+
wideLastCombinerCollable = &TProgramBuilder::WideLastCombinerWithSpilling;
1053+
}
1054+
1055+
const auto pgmReturn = pb.Collect(pb.NarrowMap((pb.*wideLastCombinerCollable)(pb.ExpandMap(pb.ToFlow(list),
10461056
[&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
10471057
[&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
10481058
[&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList {
@@ -1066,7 +1076,9 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
10661076
));
10671077

10681078
const auto graph = setup.BuildGraph(pgmReturn);
1069-
graph->GetContext().SpillerFactory = std::make_shared<TMockSpillerFactory>();
1079+
if (SPILLING) {
1080+
graph->GetContext().SpillerFactory = std::make_shared<TMockSpillerFactory>();
1081+
}
10701082
const auto iterator = graph->GetValue().GetListIterator();
10711083

10721084
std::unordered_set<TString> expected {
@@ -1079,7 +1091,6 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
10791091
NUdf::TUnboxedValue item;
10801092
while (!expected.empty()) {
10811093
UNIT_ASSERT(iterator.Next(item));
1082-
// const auto actual = item.AsStringRef();
10831094
TString actual = item.AsStringRef().Data();
10841095

10851096
auto it = expected.find(actual);
@@ -1090,8 +1101,12 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
10901101
UNIT_ASSERT(!iterator.Next(item));
10911102
}
10921103

1093-
Y_UNIT_TEST_LLVM(TestLongStringsPasstroughtRefCounting) {
1094-
TSetup<LLVM> setup;
1104+
Y_UNIT_TEST_LLVM_SPILLING(TestLongStringsPasstroughtRefCounting) {
1105+
// Currently LLVM version doesn't support spilling.
1106+
if (LLVM && SPILLING) return;
1107+
// callable WideLastCombinerWithSpilling was introduced in 49 version of runtime
1108+
if (MKQL_RUNTIME_VERSION < 49U && SPILLING) return;
1109+
TSetup<LLVM, SPILLING> setup;
10951110
TProgramBuilder& pb = *setup.PgmBuilder;
10961111

10971112
const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
@@ -1128,7 +1143,11 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
11281143

11291144
const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
11301145

1131-
const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list),
1146+
auto wideLastCombinerCollable = &TProgramBuilder::WideLastCombiner;
1147+
if (SPILLING) {
1148+
wideLastCombinerCollable = &TProgramBuilder::WideLastCombinerWithSpilling;
1149+
}
1150+
const auto pgmReturn = pb.Collect(pb.NarrowMap((pb.*wideLastCombinerCollable)(pb.ExpandMap(pb.ToFlow(list),
11321151
[&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
11331152
[&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
11341153
[&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList {
@@ -1151,22 +1170,37 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
11511170
));
11521171

11531172
const auto graph = setup.BuildGraph(pgmReturn);
1173+
if (SPILLING) {
1174+
graph->GetContext().SpillerFactory = std::make_shared<TMockSpillerFactory>();
1175+
}
11541176
const auto iterator = graph->GetValue().GetListIterator();
1177+
1178+
std::unordered_set<TString> expected {
1179+
"very long value 1 / key one / very long value 1 / key one",
1180+
"very long value 3 / key two / very long value 2 / key two",
1181+
"very long value 4 / very long key one / very long value 4 / very long key one",
1182+
"very long value 9 / very long key two / very long value 5 / very long key two"
1183+
};
1184+
11551185
NUdf::TUnboxedValue item;
1156-
UNIT_ASSERT(iterator.Next(item));
1157-
UNBOXED_VALUE_STR_EQUAL(item, "very long value 1 / key one / very long value 1 / key one");
1158-
UNIT_ASSERT(iterator.Next(item));
1159-
UNBOXED_VALUE_STR_EQUAL(item, "very long value 3 / key two / very long value 2 / key two");
1160-
UNIT_ASSERT(iterator.Next(item));
1161-
UNBOXED_VALUE_STR_EQUAL(item, "very long value 4 / very long key one / very long value 4 / very long key one");
1162-
UNIT_ASSERT(iterator.Next(item));
1163-
UNBOXED_VALUE_STR_EQUAL(item, "very long value 9 / very long key two / very long value 5 / very long key two");
1186+
while (!expected.empty()) {
1187+
UNIT_ASSERT(iterator.Next(item));
1188+
TString actual = item.AsStringRef().Data();
1189+
1190+
auto it = expected.find(actual);
1191+
UNIT_ASSERT(it != expected.end());
1192+
expected.erase(it);
1193+
}
11641194
UNIT_ASSERT(!iterator.Next(item));
11651195
UNIT_ASSERT(!iterator.Next(item));
11661196
}
11671197

1168-
Y_UNIT_TEST_LLVM(TestDoNotCalculateUnusedInput) {
1169-
TSetup<LLVM> setup;
1198+
Y_UNIT_TEST_LLVM_SPILLING(TestDoNotCalculateUnusedInput) {
1199+
// Currently LLVM version doesn't support spilling.
1200+
if (LLVM && SPILLING) return;
1201+
// callable WideLastCombinerWithSpilling was introduced in 49 version of runtime
1202+
if (MKQL_RUNTIME_VERSION < 49U && SPILLING) return;
1203+
TSetup<LLVM, SPILLING> setup;
11701204
TProgramBuilder& pb = *setup.PgmBuilder;
11711205

11721206
const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
@@ -1200,7 +1234,11 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
12001234

12011235
const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!");
12021236

1203-
const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list),
1237+
auto wideLastCombinerCollable = &TProgramBuilder::WideLastCombiner;
1238+
if (SPILLING) {
1239+
wideLastCombinerCollable = &TProgramBuilder::WideLastCombinerWithSpilling;
1240+
}
1241+
const auto pgmReturn = pb.Collect(pb.NarrowMap((pb.*wideLastCombinerCollable)(pb.ExpandMap(pb.ToFlow(list),
12041242
[&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Unwrap(pb.Nth(item, 1U), landmine, __FILE__, __LINE__, 0), pb.Nth(item, 2U)}; }),
12051243
[&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
12061244
[&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList {
@@ -1224,12 +1262,24 @@ Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
12241262
));
12251263

12261264
const auto graph = setup.BuildGraph(pgmReturn);
1265+
if (SPILLING) {
1266+
graph->GetContext().SpillerFactory = std::make_shared<TMockSpillerFactory>();
1267+
}
1268+
std::unordered_set<TString> expected {
1269+
"key one / value 2 / value 1 / value 5 / value 4",
1270+
"key two / value 4 / value 3 / value 3 / value 2"
1271+
};
1272+
12271273
const auto iterator = graph->GetValue().GetListIterator();
12281274
NUdf::TUnboxedValue item;
1229-
UNIT_ASSERT(iterator.Next(item));
1230-
UNBOXED_VALUE_STR_EQUAL(item, "key one / value 2 / value 1 / value 5 / value 4");
1231-
UNIT_ASSERT(iterator.Next(item));
1232-
UNBOXED_VALUE_STR_EQUAL(item, "key two / value 4 / value 3 / value 3 / value 2");
1275+
while (!expected.empty()) {
1276+
UNIT_ASSERT(iterator.Next(item));
1277+
TString actual = item.AsStringRef().Data();
1278+
1279+
auto it = expected.find(actual);
1280+
UNIT_ASSERT(it != expected.end());
1281+
expected.erase(it);
1282+
}
12331283
UNIT_ASSERT(!iterator.Next(item));
12341284
UNIT_ASSERT(!iterator.Next(item));
12351285
}

‎ydb/library/yql/minikql/computation/mock_spiller_factory_ut.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#pragma once
2+
3+
#include <ydb/library/yql/minikql/computation/mkql_spiller_factory.h>
4+
#include <ydb/library/yql/minikql/computation/mock_spiller_ut.h>
5+
6+
namespace NKikimr::NMiniKQL {
7+
8+
using namespace NActors;
9+
10+
class TMockSpillerFactory : public ISpillerFactory
11+
{
12+
public:
13+
ISpiller::TPtr CreateSpiller() override {
14+
return std::make_shared<TMockSpiller>();
15+
}
16+
};
17+
18+
} // namespace NKikimr::NMiniKQL

0 commit comments

Comments
 (0)