Skip to content

Commit d0d6c42

Browse files
fexolmfexolm
committed
Add decimal tests for columnshard
1 parent 2ab0fd8 commit d0d6c42

File tree

6 files changed

+115
-13
lines changed

6 files changed

+115
-13
lines changed

‎ydb/core/formats/arrow/switch/switch_type.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ TResult SwitchTypeImpl(arrow::Type::type typeId, TFunc&& f) {
7070
return f(TTypeWrapper<arrow::Time64Type>());
7171
case arrow::Type::INTERVAL_MONTHS:
7272
return f(TTypeWrapper<arrow::MonthIntervalType>());
73-
case arrow::Type::DECIMAL:
73+
case arrow::Type::DECIMAL128:
7474
return f(TTypeWrapper<arrow::Decimal128Type>());
7575
case arrow::Type::DURATION:
7676
return f(TTypeWrapper<arrow::DurationType>());

‎ydb/core/grpc_services/rpc_load_rows.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,12 @@ bool ConvertArrowToYdbPrimitive(const arrow::DataType& type, Ydb::Type& toType)
7575
case arrow::Type::DURATION:
7676
toType.set_type_id(Ydb::Type::INTERVAL);
7777
return true;
78-
case arrow::Type::DECIMAL:
79-
// TODO
80-
return false;
78+
case arrow::Type::DECIMAL: {
79+
Ydb::DecimalType* decimalType = toType.mutable_decimal_type();
80+
decimalType->set_precision(22);
81+
decimalType->set_scale(9);
82+
return true;
83+
}
8184
case arrow::Type::NA:
8285
case arrow::Type::HALF_FLOAT:
8386
case arrow::Type::FIXED_SIZE_BINARY:

‎ydb/core/kqp/ut/common/columnshard.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,8 @@ namespace NKqp {
246246
return arrow::field(name, arrow::int64(), nullable);
247247
case NScheme::NTypeIds::JsonDocument:
248248
return arrow::field(name, arrow::binary(), nullable);
249+
case NScheme::NTypeIds::Decimal:
250+
return arrow::field(name, arrow::decimal(22, 9));
249251
case NScheme::NTypeIds::Pg:
250252
switch (NPg::PgTypeIdFromTypeDesc(typeDesc)) {
251253
case INT2OID:

‎ydb/core/kqp/ut/olap/decimal_ut.cpp

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#include <ydb/core/formats/arrow/arrow_helpers.h>
2+
#include <ydb/core/kqp/ut/common/columnshard.h>
3+
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
4+
#include <ydb/core/testlib/common_helper.h>
5+
#include <ydb/core/testlib/cs_helper.h>
6+
#include <ydb/core/tx/columnshard/hooks/testing/controller.h>
7+
#include <ydb/core/tx/tx_proxy/proxy.h>
8+
9+
#include <ydb/library/binary_json/write.h>
10+
#include <ydb/library/uuid/uuid.h>
11+
#include <ydb/public/sdk/cpp/client/draft/ydb_replication.h>
12+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
13+
#include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h>
14+
#include <ydb/public/sdk/cpp/client/ydb_topic/topic.h>
15+
16+
#include <library/cpp/threading/local_executor/local_executor.h>
17+
#include <util/generic/serialized_enum.h>
18+
#include <util/string/printf.h>
19+
20+
namespace NKikimr {
21+
namespace NKqp {
22+
23+
using namespace NYdb;
24+
using namespace NYdb::NTable;
25+
26+
Y_UNIT_TEST_SUITE(KqpDecimalColumnShard) {
27+
class DecimalTestCase {
28+
public:
29+
DecimalTestCase()
30+
: TestHelper(TKikimrSettings().SetWithSampleTables(false)) {
31+
Schema = {
32+
TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false),
33+
TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64),
34+
TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::NTypeIds::Decimal),
35+
};
36+
37+
TestTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema);
38+
TestHelper.CreateTable(TestTable);
39+
}
40+
41+
TTestHelper::TUpdatesBuilder Inserter() {
42+
return TTestHelper::TUpdatesBuilder(TestTable.GetArrowSchema(Schema));
43+
}
44+
45+
void Upsert(TTestHelper::TUpdatesBuilder &inserter) {
46+
TestHelper.BulkUpsert(TestTable, inserter);
47+
}
48+
49+
void CheckQuery(const TString &query, const TString &expected) {
50+
TestHelper.ReadData(query, expected);
51+
}
52+
53+
private:
54+
TTestHelper TestHelper;
55+
56+
TVector<TTestHelper::TColumnSchema> Schema;
57+
TTestHelper::TColumnTable TestTable;
58+
};
59+
60+
Y_UNIT_TEST(SimpleQueries) {
61+
DecimalTestCase tester;
62+
63+
{
64+
TTestHelper::TUpdatesBuilder inserter = tester.Inserter();
65+
inserter.AddRow().Add(1).Add(4).Add(TDecimalValue("3.14"));
66+
inserter.AddRow().Add(2).Add(3).Add(TDecimalValue("8.16"));
67+
tester.Upsert(inserter);
68+
}
69+
tester.CheckQuery("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[[\"3.14\"];1;[4]]]");
70+
71+
{
72+
TTestHelper::TUpdatesBuilder inserter = tester.Inserter();
73+
inserter.AddRow().Add(3).Add(2).Add(TDecimalValue("8.492"));
74+
inserter.AddRow().Add(4).Add(1).Add(TDecimalValue("12.46"));
75+
tester.Upsert(inserter);
76+
}
77+
78+
tester.CheckQuery("SELECT * FROM `/Root/ColumnTableTest` order by id",
79+
"[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]");
80+
}
81+
}
82+
83+
} // namespace NKqp
84+
} // namespace NKikimr

‎ydb/core/kqp/ut/olap/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ SRCS(
2525
write_ut.cpp
2626
sparsed_ut.cpp
2727
tiering_ut.cpp
28+
decimal_ut.cpp
2829
)
2930

3031
PEERDIR(

‎ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
11
#pragma once
22

3-
#include <ydb/core/tx/columnshard/blob_cache.h>
4-
#include <ydb/core/tx/columnshard/common/snapshot.h>
5-
63
#include <ydb/core/formats/arrow/arrow_batch_builder.h>
7-
#include <ydb/core/tx/columnshard/test_helper/helper.h>
4+
#include <ydb/core/protos/tx_columnshard.pb.h>
85
#include <ydb/core/scheme/scheme_tabledefs.h>
96
#include <ydb/core/scheme/scheme_types_proto.h>
107
#include <ydb/core/testlib/tablet_helpers.h>
118
#include <ydb/core/testlib/test_client.h>
12-
#include <ydb/core/protos/tx_columnshard.pb.h>
9+
#include <ydb/core/tx/columnshard/blob_cache.h>
10+
#include <ydb/core/tx/columnshard/common/snapshot.h>
11+
#include <ydb/core/tx/columnshard/test_helper/helper.h>
12+
#include <ydb/core/tx/data_events/common/modification_type.h>
13+
#include <ydb/core/tx/long_tx_service/public/types.h>
14+
15+
#include <ydb/public/sdk/cpp/client/ydb_value/value.h>
1316
#include <ydb/services/metadata/abstract/fetcher.h>
1417

1518
#include <library/cpp/testing/unittest/registar.h>
16-
#include <ydb/core/tx/long_tx_service/public/types.h>
17-
#include <ydb/core/tx/data_events/common/modification_type.h>
1819

1920
namespace NKikimr::NOlap {
2021
struct TIndexInfo;
@@ -476,11 +477,15 @@ namespace NKikimr::NColumnShard {
476477
auto& builder = Owner.Builders[Index];
477478
auto type = builder->type();
478479

479-
NArrow::SwitchType(type->id(), [&](const auto& t) {
480+
AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("builder->type()", builder->type()->ToString());
481+
482+
Y_ABORT_UNLESS(NArrow::SwitchType(type->id(), [&](const auto& t) {
480483
using TWrap = std::decay_t<decltype(t)>;
481484
using T = typename TWrap::T;
482485
using TBuilder = typename arrow::TypeTraits<typename TWrap::T>::BuilderType;
483486

487+
AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("T", typeid(T).name());
488+
484489
auto& typedBuilder = static_cast<TBuilder&>(*builder);
485490
if constexpr (std::is_arithmetic<TData>::value) {
486491
if constexpr (arrow::has_c_type<T>::value) {
@@ -495,9 +500,16 @@ namespace NKikimr::NColumnShard {
495500
return true;
496501
}
497502
}
503+
504+
if constexpr (std::is_same<TData, NYdb::TDecimalValue>::value) {
505+
if constexpr (arrow::is_decimal128_type<T>::value) {
506+
Y_ABORT_UNLESS(typedBuilder.Append(arrow::Decimal128(data.Hi_, data.Low_)).ok());
507+
return true;
508+
}
509+
}
498510
Y_ABORT("Unknown type combination");
499511
return false;
500-
});
512+
}));
501513
return TRowBuilder(Index + 1, Owner);
502514
}
503515

0 commit comments

Comments
 (0)