Skip to content

Commit 829688f

Browse files
CyberROFLCyberROFL
authored
Merge 5d01ed0 into 2c27e6f 
2 parents 2c27e6f + 5d01ed0 commit 829688f

File tree

6 files changed

+98
-13
lines changed

6 files changed

+98
-13
lines changed

‎ydb/public/lib/ydb_cli/commands/ydb_tools.cpp

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
#include "ydb_tools.h"
22

3+
#define INCLUDE_YDB_INTERNAL_H
4+
#include <ydb/public/sdk/cpp/client/impl/ydb_internal/logger/log.h>
5+
#undef INCLUDE_YDB_INTERNAL_H
6+
37
#include <ydb/public/lib/ydb_cli/common/normalize_path.h>
48
#include <ydb/public/lib/ydb_cli/common/pg_dump_parser.h>
59
#include <ydb/public/lib/ydb_cli/dump/dump.h>
@@ -234,7 +238,10 @@ int TCommandRestore::Run(TConfig& config) {
234238
settings.Mode(NDump::TRestoreSettings::EMode::ImportData);
235239
}
236240

237-
NDump::TClient client(CreateDriver(config));
241+
auto log = std::make_shared<TLog>(CreateLogBackend("cerr", TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)));
242+
log->SetFormatter(GetPrefixLogFormatter(""));
243+
244+
NDump::TClient client(CreateDriver(config), std::move(log));
238245
ThrowOnError(client.Restore(FilePath, Path, settings));
239246

240247
return EXIT_SUCCESS;

‎ydb/public/lib/ydb_cli/dump/dump.cpp

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
#include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h>
88
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
99

10+
#include <library/cpp/logger/log.h>
11+
1012
#include <util/string/printf.h>
1113

1214
namespace NYdb {
@@ -23,8 +25,9 @@ TString DataFileName(ui32 id) {
2325

2426
class TClient::TImpl {
2527
public:
26-
explicit TImpl(const TDriver& driver)
27-
: ImportClient(driver)
28+
explicit TImpl(const TDriver& driver, std::shared_ptr<TLog>&& log)
29+
: Log(log)
30+
, ImportClient(driver)
2831
, OperationClient(driver)
2932
, SchemeClient(driver)
3033
, TableClient(driver)
@@ -37,11 +40,12 @@ class TClient::TImpl {
3740
}
3841

3942
TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) {
40-
auto client = TRestoreClient(ImportClient, OperationClient, SchemeClient, TableClient);
43+
auto client = TRestoreClient(*Log, ImportClient, OperationClient, SchemeClient, TableClient);
4144
return client.Restore(fsPath, dbPath, settings);
4245
}
4346

4447
private:
48+
std::shared_ptr<TLog> Log;
4549
NImport::TImportClient ImportClient;
4650
NOperation::TOperationClient OperationClient;
4751
NScheme::TSchemeClient SchemeClient;
@@ -60,7 +64,12 @@ TRestoreResult::TRestoreResult(TStatus&& status)
6064
}
6165

6266
TClient::TClient(const TDriver& driver)
63-
: Impl_(new TImpl(driver))
67+
: Impl_(new TImpl(driver, std::make_shared<TLog>(CreateLogBackend("cerr"))))
68+
{
69+
}
70+
71+
TClient::TClient(const TDriver& driver, std::shared_ptr<TLog>&& log)
72+
: Impl_(new TImpl(driver, std::move(log)))
6473
{
6574
}
6675

‎ydb/public/lib/ydb_cli/dump/dump.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
#include <util/generic/size_literals.h>
88

9+
class TLog;
10+
911
namespace NYdb {
1012
namespace NDump {
1113

@@ -92,6 +94,7 @@ class TClient {
9294

9395
public:
9496
explicit TClient(const TDriver& driver);
97+
explicit TClient(const TDriver& driver, std::shared_ptr<TLog>&& log);
9598

9699
TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {});
97100
TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings = {});

‎ydb/public/lib/ydb_cli/dump/restore_impl.cpp

Lines changed: 71 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
#include <ydb/public/lib/ydb_cli/common/retry_func.h>
1010
#include <ydb/public/lib/ydb_cli/dump/util/util.h>
1111

12+
#include <library/cpp/logger/log.h>
13+
1214
#include <util/generic/hash.h>
1315
#include <util/generic/hash_set.h>
1416
#include <util/generic/maybe.h>
@@ -17,6 +19,17 @@
1719
#include <util/string/builder.h>
1820
#include <util/string/join.h>
1921

22+
#define LOG_IMPL(log, level, message) \
23+
if (log.FiltrationLevel() >= level) { \
24+
log.Write(level, TStringBuilder() << message); \
25+
} \
26+
Y_SEMICOLON_GUARD
27+
28+
#define LOG_D(message) LOG_IMPL(Log, ELogPriority::TLOG_DEBUG, message)
29+
#define LOG_I(message) LOG_IMPL(Log, ELogPriority::TLOG_INFO, message)
30+
#define LOG_W(message) LOG_IMPL(Log, ELogPriority::TLOG_WARN, message)
31+
#define LOG_E(message) LOG_IMPL(Log, ELogPriority::TLOG_ERR, message)
32+
2033
namespace NYdb {
2134
namespace NDump {
2235

@@ -81,37 +94,45 @@ bool IsOperationStarted(TStatus operationStatus) {
8194
} // anonymous
8295

8396
TRestoreClient::TRestoreClient(
97+
TLog& log,
8498
TImportClient& importClient,
8599
TOperationClient& operationClient,
86100
TSchemeClient& schemeClient,
87101
TTableClient& tableClient)
88-
: ImportClient(importClient)
102+
: Log(log)
103+
, ImportClient(importClient)
89104
, OperationClient(operationClient)
90105
, SchemeClient(schemeClient)
91106
, TableClient(tableClient)
92107
{
93108
}
94109

95110
TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) {
111+
LOG_I("Restore from '" << fsPath << "' to '" << dbPath << "'");
112+
96113
// find existing items
97114
TFsPath dbBasePath = dbPath;
98115

99116
while (true) {
100-
auto result = DescribePath(SchemeClient, dbBasePath).GetStatus();
117+
auto result = DescribePath(SchemeClient, dbBasePath);
101118

102-
if (result == EStatus::SUCCESS) {
119+
if (result.GetStatus() == EStatus::SUCCESS) {
103120
break;
104121
}
105122

106-
if (result != EStatus::SCHEME_ERROR) {
123+
if (result.GetStatus() != EStatus::SCHEME_ERROR) {
124+
LOG_E("Error finding db base path: " << result.GetIssues().ToOneLineString());
107125
return Result<TRestoreResult>(EStatus::SCHEME_ERROR, "Can not find existing path");
108126
}
109127

110128
dbBasePath = dbBasePath.Parent();
111129
}
112130

131+
LOG_D("Resolved db base path: '" << dbBasePath << "'");
132+
113133
auto oldDirectoryList = RecursiveList(SchemeClient, dbBasePath);
114134
if (!oldDirectoryList.Status.IsSuccess()) {
135+
LOG_E("Error listing db base path: '" << dbBasePath << "': " << oldDirectoryList.Status.GetIssues().ToOneLineString());
115136
return Result<TRestoreResult>(EStatus::SCHEME_ERROR, "Can not list existing directory");
116137
}
117138

@@ -122,10 +143,19 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP
122143

123144
// restore
124145
auto restoreResult = RestoreFolder(fsPath, dbPath, settings, oldEntries);
125-
if (restoreResult.IsSuccess() || settings.SavePartialResult_) {
146+
if (restoreResult.IsSuccess()) {
147+
LOG_I("Restore completed successfully");
126148
return restoreResult;
127149
}
128150

151+
LOG_E("Restore failed: " << restoreResult.GetIssues().ToOneLineString());
152+
if (settings.SavePartialResult_) {
153+
LOG_I("Partial result saved");
154+
return restoreResult;
155+
}
156+
157+
LOG_I("Cleanup");
158+
129159
// cleanup
130160
auto newDirectoryList = RecursiveList(SchemeClient, dbBasePath);
131161
if (!newDirectoryList.Status.IsSuccess()) {
@@ -143,6 +173,7 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP
143173
case ESchemeEntryType::Directory: {
144174
auto result = NConsoleClient::RemoveDirectoryRecursive(SchemeClient, TableClient, fullPath, {}, true, false);
145175
if (!result.IsSuccess()) {
176+
LOG_E("Error removing directory: '" << fullPath << "': " << result.GetIssues().ToOneLineString());
146177
return restoreResult;
147178
}
148179
break;
@@ -152,11 +183,13 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP
152183
return session.DropTable(path).GetValueSync();
153184
});
154185
if (!result.IsSuccess()) {
186+
LOG_E("Error removing table: '" << fullPath << "': " << result.GetIssues().ToOneLineString());
155187
return restoreResult;
156188
}
157189
break;
158190
}
159191
default:
192+
LOG_E("Error removing unexpected object: '" << fullPath << "'");
160193
return restoreResult;
161194
}
162195
}
@@ -166,7 +199,9 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP
166199

167200
TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TString& dbPath,
168201
const TRestoreSettings& settings, const THashSet<TString>& oldEntries)
169-
{
202+
{
203+
LOG_D("Restore folder '" << fsPath << "' to '" << dbPath << "'");
204+
170205
if (!fsPath) {
171206
return Result<TRestoreResult>(EStatus::BAD_REQUEST, "Folder is not specified");
172207
}
@@ -218,28 +253,37 @@ TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TStrin
218253
TRestoreResult TRestoreClient::RestoreTable(const TFsPath& fsPath, const TString& dbPath,
219254
const TRestoreSettings& settings, const THashSet<TString>& oldEntries)
220255
{
256+
LOG_D("Process '" << fsPath << "'");
257+
221258
if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) {
222259
return Result<TRestoreResult>(EStatus::BAD_REQUEST,
223260
TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath());
224261
}
225262

263+
LOG_D("Read scheme from '" << fsPath.Child(SCHEME_FILE_NAME) << "'");
226264
auto scheme = ReadTableScheme(fsPath.Child(SCHEME_FILE_NAME));
227265
auto dumpedDesc = TableDescriptionFromProto(scheme);
228266

229267
if (dumpedDesc.GetAttributes().contains(DOC_API_TABLE_VERSION_ATTR) && settings.SkipDocumentTables_) {
268+
LOG_I("Skip document table: '" << fsPath << "'");
230269
return Result<TRestoreResult>();
231270
}
232271

233272
if (settings.DryRun_) {
234273
return CheckSchema(dbPath, dumpedDesc);
235274
}
236275

276+
LOG_I("Restore table '" << fsPath << "' to '" << dbPath << "'");
277+
237278
auto withoutIndexesDesc = TableDescriptionWithoutIndexesFromProto(scheme);
238279
auto createResult = TableClient.RetryOperationSync([&dbPath, &withoutIndexesDesc](TSession session) {
239280
return session.CreateTable(dbPath, TTableDescription(withoutIndexesDesc),
240281
TCreateTableSettings().RequestType(DOC_API_REQUEST_TYPE)).GetValueSync();
241282
});
242-
if (!createResult.IsSuccess()) {
283+
if (createResult.IsSuccess()) {
284+
LOG_D("Created '" << dbPath << "'");
285+
} else {
286+
LOG_E("Failed to create '" << dbPath << "'");
243287
return Result<TRestoreResult>(dbPath, std::move(createResult));
244288
}
245289

@@ -261,6 +305,8 @@ TRestoreResult TRestoreClient::RestoreTable(const TFsPath& fsPath, const TString
261305
}
262306

263307
TRestoreResult TRestoreClient::CheckSchema(const TString& dbPath, const TTableDescription& desc) {
308+
LOG_I("Check schema of '" << dbPath << "'");
309+
264310
TMaybe<TTableDescription> actualDesc;
265311
auto descResult = DescribeTable(TableClient, dbPath, actualDesc);
266312
if (!descResult.IsSuccess()) {
@@ -349,23 +395,28 @@ TRestoreResult TRestoreClient::RestoreData(const TFsPath& fsPath, const TString&
349395
TFsPath dataFile = fsPath.Child(DataFileName(dataFileId));
350396

351397
while (dataFile.Exists()) {
398+
LOG_D("Read data from '" << dataFile << "'");
399+
352400
TFileInput input(dataFile, settings.FileBufferSize_);
353401
TString line;
354402

355403
while (input.ReadLine(line)) {
356404
while (!accumulator->Fits(line)) {
357405
if (!accumulator->Ready(true)) {
406+
LOG_E("Error reading data from '" << dataFile << "'");
358407
return Result<TRestoreResult>(dbPath, EStatus::INTERNAL_ERROR, "Data is not ready");
359408
}
360409

361410
if (!writer->Push(accumulator->GetData(true))) {
411+
LOG_E("Error writing data to '" << dbPath << "', file: '" << dataFile << "'");
362412
return Result<TRestoreResult>(dbPath, EStatus::GENERIC_ERROR, "Cannot write data #1");
363413
}
364414
}
365415

366416
accumulator->Feed(std::move(line));
367417
if (accumulator->Ready()) {
368418
if (!writer->Push(accumulator->GetData())) {
419+
LOG_E("Error writing data to '" << dbPath << "', file: '" << dataFile << "'");
369420
return Result<TRestoreResult>(dbPath, EStatus::GENERIC_ERROR, "Cannot write data #2");
370421
}
371422
}
@@ -376,6 +427,7 @@ TRestoreResult TRestoreClient::RestoreData(const TFsPath& fsPath, const TString&
376427

377428
while (accumulator->Ready(true)) {
378429
if (!writer->Push(accumulator->GetData(true))) {
430+
LOG_E("Error writing data to '" << dbPath << "', file: '" << dataFile << "'");
379431
return Result<TRestoreResult>(dbPath, EStatus::GENERIC_ERROR, "Cannot write data #3");
380432
}
381433
}
@@ -395,6 +447,8 @@ TRestoreResult TRestoreClient::RestoreIndexes(const TString& dbPath, const TTabl
395447
continue;
396448
}
397449

450+
LOG_D("Build index '" << index.GetIndexName() << "' on '" << dbPath << "'");
451+
398452
TOperation::TOperationId buildIndexId;
399453
auto buildIndexStatus = TableClient.RetryOperationSync([&, &outId = buildIndexId](TSession session) {
400454
auto settings = TAlterTableSettings().AppendAddIndexes(index);
@@ -406,18 +460,21 @@ TRestoreResult TRestoreClient::RestoreIndexes(const TString& dbPath, const TTabl
406460
});
407461

408462
if (!IsOperationStarted(buildIndexStatus)) {
463+
LOG_E("Error building index '" << index.GetIndexName() << "' on '" << dbPath << "'");
409464
return Result<TRestoreResult>(dbPath, std::move(buildIndexStatus));
410465
}
411466

412467
auto waitForIndexBuildStatus = WaitForIndexBuild(OperationClient, buildIndexId);
413468
if (!waitForIndexBuildStatus.IsSuccess()) {
469+
LOG_E("Error building index '" << index.GetIndexName() << "' on '" << dbPath << "'");
414470
return Result<TRestoreResult>(dbPath, std::move(waitForIndexBuildStatus));
415471
}
416472

417473
auto forgetStatus = NConsoleClient::RetryFunction([&]() {
418474
return OperationClient.Forget(buildIndexId).GetValueSync();
419475
});
420476
if (!forgetStatus.IsSuccess()) {
477+
LOG_E("Error building index '" << index.GetIndexName() << "' on '" << dbPath << "'");
421478
return Result<TRestoreResult>(dbPath, std::move(forgetStatus));
422479
}
423480
}
@@ -445,18 +502,24 @@ TRestoreResult TRestoreClient::RestorePermissions(const TFsPath& fsPath, const T
445502
return Result<TRestoreResult>();
446503
}
447504

505+
LOG_D("Restore permissions to '" << dbPath << "'");
506+
448507
auto permissions = ReadPermissions(fsPath.Child(PERMISSIONS_FILE_NAME));
449508
return ModifyPermissions(SchemeClient, dbPath, TModifyPermissionsSettings(permissions));
450509
}
451510

452-
TRestoreResult TRestoreClient::RestoreEmptyDir(const TFsPath& fsPath, const TString &dbPath,
511+
TRestoreResult TRestoreClient::RestoreEmptyDir(const TFsPath& fsPath, const TString& dbPath,
453512
const TRestoreSettings& settings, const THashSet<TString>& oldEntries)
454513
{
514+
LOG_D("Process '" << fsPath << "'");
515+
455516
if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) {
456517
return Result<TRestoreResult>(EStatus::BAD_REQUEST,
457518
TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath());
458519
}
459520

521+
LOG_I("Restore empty directory '" << fsPath << "' to '" << dbPath << "'");
522+
460523
auto result = MakeDirectory(SchemeClient, dbPath);
461524
if (!result.IsSuccess()) {
462525
return result;

‎ydb/public/lib/ydb_cli/dump/restore_impl.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ class TRestoreClient {
4747

4848
public:
4949
explicit TRestoreClient(
50+
TLog& log,
5051
NImport::TImportClient& importClient,
5152
NOperation::TOperationClient& operationClient,
5253
NScheme::TSchemeClient& SchemeClient,
@@ -55,6 +56,7 @@ class TRestoreClient {
5556
TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings = {});
5657

5758
private:
59+
TLog& Log;
5860
NImport::TImportClient& ImportClient;
5961
NOperation::TOperationClient& OperationClient;
6062
NScheme::TSchemeClient& SchemeClient;

‎ydb/public/lib/ydb_cli/dump/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ SRCS(
1010

1111
PEERDIR(
1212
library/cpp/bucket_quoter
13+
library/cpp/logger
1314
library/cpp/string_utils/quote
1415
ydb/library/backup
1516
ydb/public/api/protos

0 commit comments

Comments
 (0)