Skip to content

Commit 7d436f8

Browse files
fexolmfexolm
authored
Add decimal tests for columnshard (#8959)
1 parent 618f5cf commit 7d436f8

File tree

6 files changed

+228
-12
lines changed

6 files changed

+228
-12
lines changed

‎.github/config/muted_ya.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,8 @@ ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generat
340340
ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout
341341
ydb/core/kqp/ut/query KqpStats.SysViewClientLost
342342
ydb/core/kqp/ut/olap KqpOlap.ManyColumnShards
343+
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
344+
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
343345
ydb/public/sdk/cpp/client/ydb_topic/ut TxUsage.WriteToTopic_Invalid_Session
344346
ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll
345347
ydb/tests/functional/hive test_drain.py.TestHive.test_drain_on_stop

‎ydb/core/grpc_services/rpc_load_rows.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,12 @@ bool ConvertArrowToYdbPrimitive(const arrow::DataType& type, Ydb::Type& toType)
7575
case arrow::Type::DURATION:
7676
toType.set_type_id(Ydb::Type::INTERVAL);
7777
return true;
78-
case arrow::Type::DECIMAL:
79-
// TODO
80-
return false;
78+
case arrow::Type::DECIMAL: {
79+
Ydb::DecimalType* decimalType = toType.mutable_decimal_type();
80+
decimalType->set_precision(22);
81+
decimalType->set_scale(9);
82+
return true;
83+
}
8184
case arrow::Type::NA:
8285
case arrow::Type::HALF_FLOAT:
8386
case arrow::Type::FIXED_SIZE_BINARY:

‎ydb/core/kqp/ut/common/columnshard.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,8 @@ namespace NKqp {
246246
return arrow::field(name, arrow::int64(), nullable);
247247
case NScheme::NTypeIds::JsonDocument:
248248
return arrow::field(name, arrow::binary(), nullable);
249+
case NScheme::NTypeIds::Decimal:
250+
return arrow::field(name, arrow::decimal(22, 9));
249251
case NScheme::NTypeIds::Pg:
250252
switch (NPg::PgTypeIdFromTypeDesc(typeDesc)) {
251253
case INT2OID:

‎ydb/core/kqp/ut/olap/decimal_ut.cpp

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
#include <ydb/core/formats/arrow/arrow_helpers.h>
2+
#include <ydb/core/kqp/ut/common/columnshard.h>
3+
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
4+
#include <ydb/core/testlib/common_helper.h>
5+
#include <ydb/core/testlib/cs_helper.h>
6+
#include <ydb/core/tx/columnshard/hooks/testing/controller.h>
7+
#include <ydb/core/tx/tx_proxy/proxy.h>
8+
9+
#include <ydb/library/binary_json/write.h>
10+
#include <ydb/library/uuid/uuid.h>
11+
#include <ydb/public/sdk/cpp/client/draft/ydb_replication.h>
12+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
13+
#include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h>
14+
#include <ydb/public/sdk/cpp/client/ydb_topic/topic.h>
15+
16+
#include <library/cpp/threading/local_executor/local_executor.h>
17+
#include <util/generic/serialized_enum.h>
18+
#include <util/string/printf.h>
19+
20+
namespace NKikimr {
21+
namespace NKqp {
22+
23+
using namespace NYdb;
24+
using namespace NYdb::NTable;
25+
26+
Y_UNIT_TEST_SUITE(KqpDecimalColumnShard) {
27+
class TDecimalTestCase {
28+
public:
29+
TDecimalTestCase()
30+
: TestHelper(TKikimrSettings().SetWithSampleTables(false)) {
31+
}
32+
33+
TTestHelper::TUpdatesBuilder Inserter() {
34+
return TTestHelper::TUpdatesBuilder(TestTable.GetArrowSchema(Schema));
35+
}
36+
37+
void Upsert(TTestHelper::TUpdatesBuilder& inserter) {
38+
TestHelper.BulkUpsert(TestTable, inserter);
39+
}
40+
41+
void CheckQuery(const TString& query, const TString& expected) {
42+
TestHelper.ReadData(query, expected);
43+
}
44+
45+
void PrepareTable1() {
46+
Schema = {
47+
TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false),
48+
TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64),
49+
TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::NTypeIds::Decimal),
50+
};
51+
TestTable.SetName("/Root/Table1").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema);
52+
TestHelper.CreateTable(TestTable);
53+
54+
{
55+
TTestHelper::TUpdatesBuilder inserter = Inserter();
56+
inserter.AddRow().Add(1).Add(4).Add(TDecimalValue("3.14"));
57+
inserter.AddRow().Add(2).Add(3).Add(TDecimalValue("8.16"));
58+
Upsert(inserter);
59+
}
60+
{
61+
TTestHelper::TUpdatesBuilder inserter = Inserter();
62+
inserter.AddRow().Add(4).Add(1).Add(TDecimalValue("12.46"));
63+
inserter.AddRow().Add(3).Add(2).Add(TDecimalValue("8.492"));
64+
65+
Upsert(inserter);
66+
}
67+
}
68+
69+
void PrepareTable2() {
70+
Schema = {
71+
TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false),
72+
TTestHelper::TColumnSchema().SetName("table1_id").SetType(NScheme::NTypeIds::Int64),
73+
TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::NTypeIds::Decimal),
74+
};
75+
TestTable.SetName("/Root/Table2").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema);
76+
TestHelper.CreateTable(TestTable);
77+
78+
{
79+
TTestHelper::TUpdatesBuilder inserter = Inserter();
80+
inserter.AddRow().Add(1).Add(1).Add(TDecimalValue("12.46"));
81+
inserter.AddRow().Add(2).Add(1).Add(TDecimalValue("8.16"));
82+
inserter.AddRow().Add(3).Add(2).Add(TDecimalValue("12.46"));
83+
inserter.AddRow().Add(4).Add(2).Add(TDecimalValue("8.16"));
84+
Upsert(inserter);
85+
}
86+
}
87+
88+
private:
89+
TTestHelper TestHelper;
90+
91+
TVector<TTestHelper::TColumnSchema> Schema;
92+
TTestHelper::TColumnTable TestTable;
93+
};
94+
95+
Y_UNIT_TEST(TestSimpleQueries) {
96+
TDecimalTestCase tester;
97+
tester.PrepareTable1();
98+
99+
tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE id=1", "[[[\"3.14\"];1;[4]]]");
100+
tester.CheckQuery(
101+
"SELECT * FROM `/Root/Table1` order by id", "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]");
102+
}
103+
104+
Y_UNIT_TEST(TestFilterEqual) {
105+
TDecimalTestCase tester;
106+
tester.PrepareTable1();
107+
108+
tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec == cast(\"3.14\" as decimal(22,9))", "[[[\"3.14\"];1;[4]]]");
109+
110+
tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec != cast(\"3.14\" as decimal(22,9)) order by id",
111+
"[[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]");
112+
}
113+
114+
Y_UNIT_TEST(TestFilterNulls) {
115+
TDecimalTestCase tester;
116+
tester.PrepareTable1();
117+
118+
TTestHelper::TUpdatesBuilder inserter = tester.Inserter();
119+
inserter.AddRow().Add(5).Add(5).AddNull();
120+
inserter.AddRow().Add(6).Add(6).AddNull();
121+
tester.Upsert(inserter);
122+
123+
tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec is NULL order by id", "[[#;5;[5]];[#;6;[6]]]");
124+
125+
tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec is not NULL order by id",
126+
"[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]");
127+
}
128+
129+
Y_UNIT_TEST(TestFilterCompare) {
130+
TDecimalTestCase tester;
131+
tester.PrepareTable1();
132+
133+
tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec < cast(\"12.46\" as decimal(22,9)) order by id",
134+
"[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]]]");
135+
136+
tester.CheckQuery(
137+
"SELECT * FROM `/Root/Table1` WHERE dec > cast(\"8.16\" as decimal(22,9)) order by id", "[[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]");
138+
139+
tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec <= cast(\"12.46\" as decimal(22,9)) order by id",
140+
"[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]");
141+
142+
tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec >= cast(\"8.492\" as decimal(22,9)) order by id",
143+
"[[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]");
144+
}
145+
146+
Y_UNIT_TEST(TestOrderByDecimal) {
147+
TDecimalTestCase tester;
148+
tester.PrepareTable1();
149+
150+
tester.CheckQuery(
151+
"SELECT * FROM `/Root/Table1` order by dec", "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]");
152+
}
153+
154+
Y_UNIT_TEST(TestGroupByDecimal) {
155+
TDecimalTestCase tester;
156+
tester.PrepareTable1();
157+
158+
TTestHelper::TUpdatesBuilder inserter = tester.Inserter();
159+
inserter.AddRow().Add(5).Add(12).Add(TDecimalValue("8.492"));
160+
inserter.AddRow().Add(6).Add(30).Add(TDecimalValue("12.46"));
161+
tester.Upsert(inserter);
162+
163+
tester.CheckQuery("SELECT dec, count(*) FROM `/Root/Table1` group by dec order by dec",
164+
"[[[\"3.14\"];1u];[[\"8.16\"];1u];[[\"8.492\"];2u];[[\"12.46\"];2u]]");
165+
}
166+
167+
Y_UNIT_TEST(TestAggregation) {
168+
TDecimalTestCase tester;
169+
tester.PrepareTable1();
170+
tester.CheckQuery("SELECT min(dec) FROM `/Root/Table1`", "[[[\"3.14\"]]]");
171+
tester.CheckQuery("SELECT max(dec) FROM `/Root/Table1`", "[[[\"12.46\"]]]");
172+
tester.CheckQuery("SELECT sum(dec) FROM `/Root/Table1`", "[[[\"32.252\"]]]");
173+
}
174+
175+
Y_UNIT_TEST(TestJoinById) {
176+
TDecimalTestCase tester;
177+
tester.PrepareTable1();
178+
tester.PrepareTable2();
179+
180+
tester.CheckQuery(
181+
"SELECT t1.id, t1.dec, t2.dec FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.id = t2.table1_id order by t1.id, t1.dec, "
182+
"t2.dec",
183+
R"([[1;["3.14"];["8.16"]];[1;["3.14"];["12.46"]];[2;["8.16"];["8.16"]];[2;["8.16"];["12.46"]]])");
184+
}
185+
186+
Y_UNIT_TEST(TestJoinByDecimal) {
187+
TDecimalTestCase tester;
188+
tester.PrepareTable1();
189+
tester.PrepareTable2();
190+
191+
tester.CheckQuery(
192+
"SELECT t1.id, t2.id, t1.dec FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.dec = t2.dec order by t1.id, t2.id, t1.dec",
193+
R"([[2;2;["8.16"]];[2;4;["8.16"]];[4;1;["12.46"]];[4;3;["12.46"]]])");
194+
}
195+
}
196+
197+
} // namespace NKqp
198+
} // namespace NKikimr

‎ydb/core/kqp/ut/olap/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ SRCS(
2525
write_ut.cpp
2626
sparsed_ut.cpp
2727
tiering_ut.cpp
28+
decimal_ut.cpp
2829
)
2930

3031
PEERDIR(

‎ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
11
#pragma once
22

3-
#include <ydb/core/tx/columnshard/blob_cache.h>
4-
#include <ydb/core/tx/columnshard/common/snapshot.h>
5-
63
#include <ydb/core/formats/arrow/arrow_batch_builder.h>
7-
#include <ydb/core/tx/columnshard/test_helper/helper.h>
4+
#include <ydb/core/protos/tx_columnshard.pb.h>
85
#include <ydb/core/scheme/scheme_tabledefs.h>
96
#include <ydb/core/scheme/scheme_types_proto.h>
107
#include <ydb/core/testlib/tablet_helpers.h>
118
#include <ydb/core/testlib/test_client.h>
12-
#include <ydb/core/protos/tx_columnshard.pb.h>
9+
#include <ydb/core/tx/columnshard/blob_cache.h>
10+
#include <ydb/core/tx/columnshard/common/snapshot.h>
11+
#include <ydb/core/tx/columnshard/test_helper/helper.h>
12+
#include <ydb/core/tx/data_events/common/modification_type.h>
13+
#include <ydb/core/tx/long_tx_service/public/types.h>
14+
15+
#include <ydb/public/sdk/cpp/client/ydb_value/value.h>
1316
#include <ydb/services/metadata/abstract/fetcher.h>
1417

1518
#include <library/cpp/testing/unittest/registar.h>
16-
#include <ydb/core/tx/long_tx_service/public/types.h>
17-
#include <ydb/core/tx/data_events/common/modification_type.h>
1819

1920
namespace NKikimr::NOlap {
2021
struct TIndexInfo;
@@ -476,11 +477,13 @@ namespace NKikimr::NColumnShard {
476477
auto& builder = Owner.Builders[Index];
477478
auto type = builder->type();
478479

479-
NArrow::SwitchType(type->id(), [&](const auto& t) {
480+
Y_ABORT_UNLESS(NArrow::SwitchType(type->id(), [&](const auto& t) {
480481
using TWrap = std::decay_t<decltype(t)>;
481482
using T = typename TWrap::T;
482483
using TBuilder = typename arrow::TypeTraits<typename TWrap::T>::BuilderType;
483484

485+
AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("T", typeid(T).name());
486+
484487
auto& typedBuilder = static_cast<TBuilder&>(*builder);
485488
if constexpr (std::is_arithmetic<TData>::value) {
486489
if constexpr (arrow::has_c_type<T>::value) {
@@ -495,9 +498,16 @@ namespace NKikimr::NColumnShard {
495498
return true;
496499
}
497500
}
501+
502+
if constexpr (std::is_same<TData, NYdb::TDecimalValue>::value) {
503+
if constexpr (arrow::is_decimal128_type<T>::value) {
504+
Y_ABORT_UNLESS(typedBuilder.Append(arrow::Decimal128(data.Hi_, data.Low_)).ok());
505+
return true;
506+
}
507+
}
498508
Y_ABORT("Unknown type combination");
499509
return false;
500-
});
510+
}));
501511
return TRowBuilder(Index + 1, Owner);
502512
}
503513

0 commit comments

Comments
 (0)