Skip to content

Commit e39226c

Browse files
dcherednikdcherednik
authored
Add backup/restore functional test using UUID type (#6619)
1 parent f04a115 commit e39226c

File tree

3 files changed

+189
-0
lines changed

3 files changed

+189
-0
lines changed

‎ydb/tests/functional/backup/main.cpp

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
#include <util/system/env.h>
2+
#include <library/cpp/testing/unittest/registar.h>
3+
4+
#include <ydb/library/testlib/s3_recipe_helper/s3_recipe_helper.h>
5+
6+
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
7+
#include <ydb/public/sdk/cpp/client/ydb_export/export.h>
8+
#include <ydb/public/sdk/cpp/client/ydb_import/import.h>
9+
#include <ydb/public/sdk/cpp/client/ydb_operation/operation.h>
10+
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
11+
#include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h>
12+
#include <ydb/public/lib/yson_value/ydb_yson_value.h>
13+
#include <library/cpp/yson/writer.h>
14+
15+
#include <library/cpp/threading/local_executor/local_executor.h>
16+
17+
using namespace NYdb;
18+
using namespace NYdb::NTable;
19+
20+
namespace {
21+
template<typename TOp>
22+
void WaitOp(TMaybe<TOperation>& op, NOperation::TOperationClient& opClient) {
23+
int attempt = 20;
24+
while (--attempt) {
25+
op = opClient.Get<TOp>(op->Id()).GetValueSync();
26+
if (op->Ready()) {
27+
break;
28+
}
29+
Sleep(TDuration::Seconds(1));
30+
}
31+
UNIT_ASSERT_C(attempt, "Unable to wait completion of backup");
32+
}
33+
34+
TString ReformatYson(const TString& yson) {
35+
TStringStream ysonInput(yson);
36+
TStringStream output;
37+
NYson::ReformatYsonStream(&ysonInput, &output, NYson::EYsonFormat::Text);
38+
return output.Str();
39+
}
40+
41+
void CompareYson(const TString& expected, const TString& actual) {
42+
UNIT_ASSERT_NO_DIFF(ReformatYson(expected), ReformatYson(actual));
43+
}
44+
}
45+
46+
Y_UNIT_TEST_SUITE(Backup)
47+
{
48+
Y_UNIT_TEST(UuidValue)
49+
{
50+
TString connectionString = GetEnv("YDB_ENDPOINT") + "/?database=" + GetEnv("YDB_DATABASE");
51+
auto config = TDriverConfig(connectionString);
52+
auto driver = TDriver(config);
53+
auto tableClient = TTableClient(driver);
54+
auto session = tableClient.GetSession().GetValueSync().GetSession();
55+
56+
{
57+
auto res = session.ExecuteSchemeQuery(R"(
58+
CREATE TABLE `/local/ProducerUuidValue` (
59+
Key Uint32,
60+
Value1 Uuid,
61+
Value2 Uuid NOT NULL,
62+
PRIMARY KEY (Key)
63+
);
64+
)").GetValueSync();
65+
UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
66+
}
67+
68+
{
69+
auto sessionResult = tableClient.GetSession().GetValueSync();
70+
UNIT_ASSERT_C(sessionResult.IsSuccess(), sessionResult.GetIssues().ToString());
71+
auto s = sessionResult.GetSession();
72+
73+
{
74+
const TString query = "UPSERT INTO ProducerUuidValue (Key, Value1, Value2) VALUES"
75+
"(1, "
76+
"CAST(\"5b99a330-04ef-4f1a-9b64-ba6d5f44ea01\" as Uuid), "
77+
"UNWRAP(CAST(\"5b99a330-04ef-4f1a-9b64-ba6d5f44ea02\" as Uuid)"
78+
"));";
79+
auto res = s.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync();
80+
UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
81+
}
82+
}
83+
84+
const TString bucketName = "bbb";
85+
NTestUtils::CreateBucket(bucketName);
86+
87+
auto fillS3Settings = [bucketName](auto& settings) {
88+
settings.Endpoint(GetEnv("S3_ENDPOINT"));
89+
settings.Bucket(bucketName);
90+
settings.AccessKey("minio");
91+
settings.SecretKey("minio123");
92+
};
93+
94+
{
95+
NExport::TExportToS3Settings settings;
96+
fillS3Settings(settings);
97+
98+
settings.AppendItem({"/local/ProducerUuidValue", "ProducerUuidValueBackup"});
99+
100+
auto exportClient = NExport::TExportClient(driver);
101+
auto operationClient = NOperation::TOperationClient(driver);
102+
103+
const auto backupOp = exportClient.ExportToS3(settings).GetValueSync();
104+
105+
if (backupOp.Ready()) {
106+
UNIT_ASSERT_C(backupOp.Status().IsSuccess(), backupOp.Status().GetIssues().ToString());
107+
} else {
108+
TMaybe<TOperation> op = backupOp;
109+
WaitOp<NExport::TExportToS3Response>(op, operationClient);
110+
UNIT_ASSERT_C(op->Status().IsSuccess(), op->Status().GetIssues().ToString());
111+
}
112+
}
113+
114+
auto ob = NTestUtils::GetObjectKeys(bucketName);
115+
std::sort(ob.begin(), ob.end());
116+
UNIT_ASSERT_VALUES_EQUAL(ob.size(), 3);
117+
UNIT_ASSERT_VALUES_EQUAL(ob[0], "ProducerUuidValueBackup/data_00.csv");
118+
UNIT_ASSERT_VALUES_EQUAL(ob[1], "ProducerUuidValueBackup/metadata.json");
119+
UNIT_ASSERT_VALUES_EQUAL(ob[2], "ProducerUuidValueBackup/scheme.pb");
120+
121+
{
122+
NImport::TImportFromS3Settings settings;
123+
fillS3Settings(settings);
124+
125+
settings.AppendItem({"ProducerUuidValueBackup", "/local/restore"});
126+
127+
auto importClient = NImport::TImportClient(driver);
128+
auto operationClient = NOperation::TOperationClient(driver);
129+
130+
const auto restoreOp = importClient.ImportFromS3(settings).GetValueSync();
131+
132+
if (restoreOp.Ready()) {
133+
UNIT_ASSERT_C(restoreOp.Status().IsSuccess(), restoreOp.Status().GetIssues().ToString());
134+
} else {
135+
TMaybe<TOperation> op = restoreOp;
136+
WaitOp<NImport::TImportFromS3Response>(op, operationClient);
137+
UNIT_ASSERT_C(op->Status().IsSuccess(), op->Status().GetIssues().ToString());
138+
}
139+
}
140+
141+
{
142+
auto sessionResult = tableClient.GetSession().GetValueSync();
143+
UNIT_ASSERT_C(sessionResult.IsSuccess(), sessionResult.GetIssues().ToString());
144+
auto s = sessionResult.GetSession();
145+
146+
{
147+
const TString query = "SELECT * FROM `/local/restore`;";
148+
auto res = s.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).GetValueSync();
149+
UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
150+
151+
auto yson = NYdb::FormatResultSetYson(res.GetResultSet(0));
152+
153+
const TString& expected = "[[[1u];[\"5b99a330-04ef-4f1a-9b64-ba6d5f44ea01\"];\"5b99a330-04ef-4f1a-9b64-ba6d5f44ea02\"]]";
154+
CompareYson(expected, yson);
155+
}
156+
}
157+
}
158+
}
159+

‎ydb/tests/functional/backup/ya.make

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
UNITTEST()
2+
3+
ENV(YDB_USE_IN_MEMORY_PDISKS=true)
4+
5+
ENV(YDB_ERASURE=block_4-2)
6+
7+
PEERDIR(
8+
library/cpp/threading/local_executor
9+
library/cpp/yson
10+
ydb/library/testlib/s3_recipe_helper
11+
ydb/public/sdk/cpp/client/ydb_export
12+
ydb/public/sdk/cpp/client/ydb_table
13+
ydb/public/sdk/cpp/client/ydb_operation
14+
ydb/public/sdk/cpp/client/draft
15+
ydb/public/lib/yson_value
16+
)
17+
18+
SRCS(
19+
main.cpp
20+
)
21+
22+
INCLUDE(${ARCADIA_ROOT}/ydb/public/tools/ydb_recipe/recipe.inc)
23+
INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/s3_recipe/recipe.inc)
24+
25+
SIZE(MEDIUM)
26+
27+
REQUIREMENTS(ram:16)
28+
29+
END()

‎ydb/tests/functional/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ RECURSE(
22
api
33
audit
44
autoconfig
5+
backup
56
blobstorage
67
canonical
78
clickbench

0 commit comments

Comments
 (0)