Skip to content

Commit aa85c35

Browse files
dcherednikdcherednik
authored
[CDC] Do not lose presition during float/double to json serialization… (#7737)
1 parent 65add49 commit aa85c35

File tree

4 files changed

+150
-4
lines changed

4 files changed

+150
-4
lines changed

‎ydb/core/tx/datashard/change_record_cdc_serializer.cpp

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,16 @@ class TJsonSerializer: public TBaseSerializer {
9191
friend class TChangeRecord; // used in GetPartitionKey()
9292

9393
static NJson::TJsonWriterConfig DefaultJsonConfig() {
94-
NJson::TJsonWriterConfig jsonConfig;
95-
jsonConfig.ValidateUtf8 = false;
96-
jsonConfig.WriteNanAsString = true;
97-
return jsonConfig;
94+
constexpr ui32 doubleNDigits = std::numeric_limits<double>::max_digits10;
95+
constexpr ui32 floatNDigits = std::numeric_limits<float>::max_digits10;
96+
constexpr EFloatToStringMode floatMode = EFloatToStringMode::PREC_NDIGITS;
97+
return NJson::TJsonWriterConfig {
98+
.DoubleNDigits = doubleNDigits,
99+
.FloatNDigits = floatNDigits,
100+
.FloatToStringMode = floatMode,
101+
.ValidateUtf8 = false,
102+
.WriteNanAsString = true,
103+
};
98104
}
99105

100106
protected:

‎ydb/tests/functional/replication/main.cpp

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#include <util/system/env.h>
2+
#include <library/cpp/testing/unittest/registar.h>
3+
4+
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
5+
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
6+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
7+
#include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h>
8+
9+
#include <library/cpp/threading/local_executor/local_executor.h>
10+
11+
using namespace NYdb;
12+
using namespace NYdb::NTable;
13+
14+
namespace {
15+
16+
std::pair<ui64, Ydb::ResultSet> DoRead(TSession& s, const TString& table) {
17+
auto res = s.ExecuteDataQuery(
18+
Sprintf("SELECT * FROM `/local/%s`; SELECT COUNT(*) AS __count FROM `/local/%s`;",
19+
table.data(), table.data()), TTxControl::BeginTx().CommitTx()).GetValueSync();
20+
UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
21+
auto rs = NYdb::TResultSetParser(res.GetResultSet(1));
22+
UNIT_ASSERT(rs.TryNextRow());
23+
auto count = rs.ColumnParser("__count").GetUint64();
24+
25+
const auto proto = NYdb::TProtoAccessor::GetProto(res.GetResultSet(0));
26+
return {count, proto};
27+
}
28+
29+
} // namespace
30+
31+
Y_UNIT_TEST_SUITE(Replication)
32+
{
33+
Y_UNIT_TEST(Types)
34+
{
35+
TString connectionString = GetEnv("YDB_ENDPOINT") + "/?database=" + GetEnv("YDB_DATABASE");
36+
auto config = TDriverConfig(connectionString);
37+
auto driver = TDriver(config);
38+
auto tableClient = TTableClient(driver);
39+
auto session = tableClient.GetSession().GetValueSync().GetSession();
40+
41+
{
42+
auto res = session.ExecuteSchemeQuery(R"(
43+
CREATE TABLE `/local/ProducerUuidValue` (
44+
Key Uint32,
45+
v01 Uuid,
46+
v02 Uuid NOT NULL,
47+
v03 Double,
48+
PRIMARY KEY (Key)
49+
);
50+
)").GetValueSync();
51+
UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
52+
}
53+
54+
{
55+
auto sessionResult = tableClient.GetSession().GetValueSync();
56+
UNIT_ASSERT_C(sessionResult.IsSuccess(), sessionResult.GetIssues().ToString());
57+
auto s = sessionResult.GetSession();
58+
59+
{
60+
const TString query = "UPSERT INTO ProducerUuidValue (Key,v01,v02,v03) VALUES"
61+
"(1, "
62+
"CAST(\"5b99a330-04ef-4f1a-9b64-ba6d5f44ea01\" as Uuid), "
63+
"UNWRAP(CAST(\"5b99a330-04ef-4f1a-9b64-ba6d5f44ea02\" as Uuid)), "
64+
"CAST(\"311111111113.222222223\" as Double) "
65+
");";
66+
auto res = s.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync();
67+
UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
68+
}
69+
70+
{
71+
const TString query = Sprintf("CREATE ASYNC REPLICATION `replication` FOR"
72+
"`ProducerUuidValue` AS `ConsumerUuidValue`"
73+
"WITH ("
74+
"CONNECTION_STRING = 'grpc://%s',"
75+
"TOKEN = 'root@builtin'"
76+
");", connectionString.data());
77+
auto res = s.ExecuteSchemeQuery(query).GetValueSync();
78+
UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
79+
}
80+
// TODO: Make CREATE ASYNC REPLICATION to be a sync call
81+
Sleep(TDuration::Seconds(10));
82+
}
83+
84+
NYdb::NTable::TExecDataQuerySettings execSettings;
85+
execSettings.KeepInQueryCache(true);
86+
87+
auto sessionResult = tableClient.GetSession().GetValueSync();
88+
UNIT_ASSERT_C(sessionResult.IsSuccess(), sessionResult.GetIssues().ToString());
89+
90+
auto s = sessionResult.GetSession();
91+
TUuidValue expectedV1("5b99a330-04ef-4f1a-9b64-ba6d5f44ea01");
92+
TUuidValue expectedV2("5b99a330-04ef-4f1a-9b64-ba6d5f44ea02");
93+
double expectedV3 = 311111111113.222222223;
94+
ui32 attempt = 10;
95+
while (--attempt) {
96+
auto res = DoRead(s, "ConsumerUuidValue");
97+
if (res.first == 1) {
98+
const Ydb::ResultSet& proto = res.second;
99+
UNIT_ASSERT_VALUES_EQUAL(proto.rows(0).items(0).uint32_value(), 1);
100+
UNIT_ASSERT_VALUES_EQUAL(proto.rows(0).items(1).low_128(), expectedV1.Buf_.Halfs[0]);
101+
UNIT_ASSERT_VALUES_EQUAL(proto.rows(0).items(1).high_128(), expectedV1.Buf_.Halfs[1]);
102+
UNIT_ASSERT_VALUES_EQUAL(proto.rows(0).items(2).low_128(), expectedV2.Buf_.Halfs[0]);
103+
UNIT_ASSERT_VALUES_EQUAL(proto.rows(0).items(2).high_128(), expectedV2.Buf_.Halfs[1]);
104+
UNIT_ASSERT_DOUBLES_EQUAL(proto.rows(0).items(3).double_value(), expectedV3, 0.0001);
105+
break;
106+
}
107+
Sleep(TDuration::Seconds(1));
108+
}
109+
110+
UNIT_ASSERT_C(attempt, "Unable to wait replication result");
111+
}
112+
}
113+

‎ydb/tests/functional/replication/ya.make

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
UNITTEST()
2+
3+
ENV(YDB_USE_IN_MEMORY_PDISKS=true)
4+
5+
ENV(YDB_ERASURE=block_4-2)
6+
7+
PEERDIR(
8+
library/cpp/threading/local_executor
9+
ydb/public/sdk/cpp/client/ydb_table
10+
ydb/public/sdk/cpp/client/ydb_proto
11+
ydb/public/sdk/cpp/client/draft
12+
)
13+
14+
SRCS(
15+
main.cpp
16+
)
17+
18+
INCLUDE(${ARCADIA_ROOT}/ydb/public/tools/ydb_recipe/recipe.inc)
19+
20+
SIZE(MEDIUM)
21+
22+
IF (SANITIZER_TYPE)
23+
REQUIREMENTS(ram:16)
24+
ENDIF()
25+
26+
END()

‎ydb/tests/functional/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ RECURSE(
1818
query_cache
1919
rename
2020
restarts
21+
replication
2122
scheme_shard
2223
scheme_tests
2324
script_execution

0 commit comments

Comments
 (0)