Encapsulate BackupFolder() in DumpClient (#10804)

Author: CyberROFL

ydb/library/backup/util.cpp

@@ -20,11 +20,11 @@ struct TLog {
     {}
 };
 
-void SetLog(std::shared_ptr<::TLog>&& log) {
-    Singleton<TLog>()->Log = std::move(log);
+void SetLog(const std::shared_ptr<::TLog>& log) {
+    Singleton<TLog>()->Log = log;
 }
 
-std::shared_ptr<::TLog>& GetLog() {
+const std::shared_ptr<::TLog>& GetLog() {
     return Singleton<TLog>()->Log;
 }
 

ydb/library/backup/util.h

@@ -18,8 +18,8 @@
 namespace NYdb {
 
 namespace NBackup {
-    void SetLog(std::shared_ptr<::TLog>&& log);
-    std::shared_ptr<::TLog>& GetLog();
+    void SetLog(const std::shared_ptr<::TLog>& log);
+    const std::shared_ptr<::TLog>& GetLog();
 }
 
 // Retrive path relative to database root from absolute

ydb/public/lib/ydb_cli/commands/ydb_tools.cpp

@@ -7,9 +7,11 @@
 #include <ydb/public/lib/ydb_cli/common/normalize_path.h>
 #include <ydb/public/lib/ydb_cli/common/pg_dump_parser.h>
 #include <ydb/public/lib/ydb_cli/dump/dump.h>
-#include <ydb/library/backup/backup.h>
 #include <ydb/library/backup/util.h>
 
+#include <library/cpp/regex/pcre/regexp.h>
+
+#include <util/generic/serialized_enum.h>
 #include <util/stream/format.h>
 #include <util/string/split.h>
 
@@ -58,28 +60,31 @@ void TCommandDump::Config(TConfig& config) {
     config.Opts->AddLongOption('o', "output", "[Required] Path in a local filesystem to a directory to place dump into."
             " Directory should either not exist or be empty.")
         .StoreResult(&FilePath);
+
+    NDump::TDumpSettings defaults;
+
     config.Opts->AddLongOption("scheme-only", "Dump only scheme including ACL and owner")
-        .StoreTrue(&IsSchemeOnly);
+        .DefaultValue(defaults.SchemaOnly_).StoreTrue(&IsSchemeOnly);
     config.Opts->AddLongOption("avoid-copy", "Avoid copying."
             " By default, YDB makes a copy of a table before dumping it to reduce impact on workload and ensure consistency.\n"
             "In some cases (e.g. for tables with external blobs) copying should be disabled.")
-        .StoreTrue(&AvoidCopy);
+        .DefaultValue(defaults.AvoidCopy_).StoreTrue(&AvoidCopy);
     config.Opts->AddLongOption("save-partial-result", "Do not remove partial dump result."
             " If this option is not enabled, all files that have already been created will be removed in case of error.")
-        .StoreTrue(&SavePartialResult);
+        .DefaultValue(defaults.SavePartialResult_).StoreTrue(&SavePartialResult);
     config.Opts->AddLongOption("preserve-pool-kinds", "Preserve storage pool kind settings."
             " If this option is enabled, storage pool kind will be saved to dump."
             " In this case, if there will be no such storage pool kind in database on restore, error will occur."
             " By default this option is disabled and any existing storage pool kind will be used on restore.")
-        .StoreTrue(&PreservePoolKinds);
+        .DefaultValue(defaults.PreservePoolKinds_).StoreTrue(&PreservePoolKinds);
     config.Opts->AddLongOption("consistency-level", "Consistency level."
             " Options: database, table\n"
             "database - take one consistent snapshot of all tables specified for dump."
             " Takes more time and is more likely to impact workload;\n"
             "table - take consistent snapshot per each table independently.")
-        .DefaultValue("database").StoreResult(&ConsistencyLevel);
+        .DefaultValue(defaults.ConsistencyLevel_).StoreResult(&ConsistencyLevel);
     config.Opts->AddLongOption("ordered", "Preserve order by primary key in backup files.")
-            .StoreTrue(&Ordered);
+        .DefaultValue(defaults.Ordered_).StoreTrue(&Ordered);
 }
 
 void TCommandDump::Parse(TConfig& config) {
@@ -88,31 +93,28 @@ void TCommandDump::Parse(TConfig& config) {
 }
 
 int TCommandDump::Run(TConfig& config) {
-
-    bool useConsistentCopyTable;
-    if (ConsistencyLevel == "database") {
-        useConsistentCopyTable = true;
-    } else if (ConsistencyLevel == "table") {
-        useConsistentCopyTable = false;
-    } else {
-        throw yexception() << "Incorrect consistency level. Available options: \"database\", \"table\"" << Endl;
+    NDump::TDumpSettings::EConsistencyLevel consistencyLevel;
+    if (!TryFromString<NDump::TDumpSettings::EConsistencyLevel>(ConsistencyLevel, consistencyLevel)) {
+        throw yexception() << "Incorrect consistency level."
+            " Available options: " << GetEnumAllNames<NDump::TDumpSettings::EConsistencyLevel>();
     }
 
+    auto settings = NDump::TDumpSettings()
+        .Database(config.Database)
+        .ExclusionPatterns(std::move(ExclusionPatterns))
+        .SchemaOnly(IsSchemeOnly)
+        .ConsistencyLevel(consistencyLevel)
+        .AvoidCopy(AvoidCopy)
+        .SavePartialResult(SavePartialResult)
+        .PreservePoolKinds(PreservePoolKinds)
+        .Ordered(Ordered);
+
     auto log = std::make_shared<TLog>(CreateLogBackend("cerr", TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)));
     log->SetFormatter(GetPrefixLogFormatter(""));
-    NYdb::NBackup::SetLog(std::move(log));
-
-    try {
-        TString relPath = NYdb::RelPathFromAbsolute(config.Database, Path);
-        NYdb::NBackup::BackupFolder(CreateDriver(config), config.Database, relPath, FilePath, ExclusionPatterns,
-            IsSchemeOnly, useConsistentCopyTable, AvoidCopy, SavePartialResult, PreservePoolKinds, Ordered);
-    } catch (const NYdb::NBackup::TYdbErrorException& e) {
-        e.LogToStderr();
-        return EXIT_FAILURE;
-    } catch (const yexception& e) {
-        Cerr << "General error, what# " << e.what() << Endl;
-        return EXIT_FAILURE;
-    }
+
+    NDump::TClient client(CreateDriver(config), std::move(log));
+    ThrowOnError(client.Dump(Path, FilePath, settings));
+
     return EXIT_SUCCESS;
 }
 

ydb/public/lib/ydb_cli/commands/ydb_tools.h

@@ -7,7 +7,7 @@
 #include <ydb/public/lib/ydb_cli/common/examples.h>
 #include <ydb/public/lib/ydb_cli/common/parseable_struct.h>
 
-#include <library/cpp/regex/pcre/regexp.h>
+class TRegExMatch;
 
 namespace NYdb {
 namespace NConsoleClient {

ydb/public/lib/ydb_cli/dump/dump.cpp

@@ -2,10 +2,7 @@
 #include "dump_impl.h"
 #include "restore_impl.h"
 
-#include <ydb/public/sdk/cpp/client/ydb_import/import.h>
-#include <ydb/public/sdk/cpp/client/ydb_operation/operation.h>
-#include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h>
-#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
+#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
 
 #include <library/cpp/logger/log.h>
 
@@ -26,30 +23,24 @@ TString DataFileName(ui32 id) {
 class TClient::TImpl {
 public:
     explicit TImpl(const TDriver& driver, std::shared_ptr<TLog>&& log)
-        : Log(log)
-        , ImportClient(driver)
-        , OperationClient(driver)
-        , SchemeClient(driver)
-        , TableClient(driver)
+        : Driver(driver)
+        , Log(std::move(log))
     {
     }
 
     TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings) {
-        auto client = TDumpClient(SchemeClient, TableClient);
+        auto client = TDumpClient(Driver, Log);
         return client.Dump(dbPath, fsPath, settings);
     }
 
     TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) {
-        auto client = TRestoreClient(*Log, ImportClient, OperationClient, SchemeClient, TableClient);
+        auto client = TRestoreClient(Driver, *Log);
         return client.Restore(fsPath, dbPath, settings);
     }
 
 private:
+    const TDriver Driver;
     std::shared_ptr<TLog> Log;
-    NImport::TImportClient ImportClient;
-    NOperation::TOperationClient OperationClient;
-    NScheme::TSchemeClient SchemeClient;
-    NTable::TTableClient TableClient;
 
 }; // TImpl
 

ydb/public/lib/ydb_cli/dump/dump.h

@@ -2,13 +2,18 @@
 
 #include <ydb/public/sdk/cpp/client/ydb_types/status/status.h>
 #include <ydb/public/sdk/cpp/client/ydb_types/fluent_settings_helpers.h>
-#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
+#include <ydb/public/sdk/cpp/client/ydb_types/request_settings.h>
+
+#include <library/cpp/regex/pcre/regexp.h>
 
 #include <util/generic/size_literals.h>
 
 class TLog;
 
 namespace NYdb {
+
+class TDriver;
+
 namespace NDump {
 
 extern const char SCHEME_FILE_NAME[10];
@@ -22,6 +27,34 @@ TString DataFileName(ui32 id);
 struct TDumpSettings: public TOperationRequestSettings<TDumpSettings> {
     using TSelf = TDumpSettings;
 
+    enum class EConsistencyLevel {
+        Table /* "table" */,
+        Database /* "database" */,
+    };
+
+    TVector<TRegExMatch> ExclusionPatterns_;
+    TSelf& ExclusionPatterns(TVector<TRegExMatch>&& value) {
+        ExclusionPatterns_ = std::move(value);
+        return *this;
+    }
+
+    FLUENT_SETTING(TString, Database);
+    FLUENT_SETTING_DEFAULT(bool, SchemaOnly, false);
+    FLUENT_SETTING_DEFAULT(EConsistencyLevel, ConsistencyLevel, EConsistencyLevel::Database);
+    FLUENT_SETTING_DEFAULT(bool, AvoidCopy, false);
+    FLUENT_SETTING_DEFAULT(bool, SavePartialResult, false);
+    FLUENT_SETTING_DEFAULT(bool, PreservePoolKinds, false);
+    FLUENT_SETTING_DEFAULT(bool, Ordered, false);
+
+    bool UseConsistentCopyTable() const {
+        switch (ConsistencyLevel_) {
+            case EConsistencyLevel::Table:
+                return false;
+            case EConsistencyLevel::Database:
+                return true;
+        }
+    }
+
 }; // TDumpSettings
 
 class TDumpResult: public TStatus {

ydb/public/lib/ydb_cli/dump/dump_impl.cpp

@@ -1,24 +1,39 @@
 #include "dump_impl.h"
 
+#include <ydb/library/backup/backup.h>
+#include <ydb/library/backup/util.h>
 #include <ydb/public/lib/ydb_cli/dump/util/util.h>
 
 namespace NYdb {
 namespace NDump {
 
-TDumpClient::TDumpClient(NScheme::TSchemeClient& schemeClient, NTable::TTableClient& tableClient)
-    : SchemeClient(schemeClient)
-    , TableClient(tableClient)
+TDumpClient::TDumpClient(const TDriver& driver, const std::shared_ptr<TLog>& log)
+    : Driver(driver)
+    , Log(log)
 {
-    Y_UNUSED(SchemeClient);
-    Y_UNUSED(TableClient);
 }
 
 TDumpResult TDumpClient::Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings) {
-    Y_UNUSED(dbPath);
-    Y_UNUSED(TableClient);
-    Y_UNUSED(fsPath);
-    Y_UNUSED(settings);
-    return Result<TDumpResult>(EStatus::CLIENT_CALL_UNIMPLEMENTED, "Not implemented");
+    try {
+        NBackup::SetLog(Log);
+        NBackup::BackupFolder(
+            Driver,
+            settings.Database_,
+            RelPathFromAbsolute(settings.Database_, dbPath),
+            fsPath,
+            settings.ExclusionPatterns_,
+            settings.SchemaOnly_,
+            settings.UseConsistentCopyTable(),
+            settings.AvoidCopy_,
+            settings.SavePartialResult_,
+            settings.PreservePoolKinds_,
+            settings.Ordered_);
+        return Result<TDumpResult>();
+    } catch (NBackup::TYdbErrorException& e) {
+        return TDumpResult(std::move(e.Status));
+    } catch (const yexception& e) {
+        return Result<TDumpResult>(EStatus::INTERNAL_ERROR, e.what());
+    }
 }
 
 } // NDump

ydb/public/lib/ydb_cli/dump/dump_impl.h

@@ -2,21 +2,18 @@
 
 #include "dump.h"
 
-#include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h>
-#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
-
 namespace NYdb {
 namespace NDump {
 
 class TDumpClient {
 public:
-    explicit TDumpClient(NScheme::TSchemeClient& schemeClient, NTable::TTableClient& tableClient);
+    explicit TDumpClient(const TDriver& driver, const std::shared_ptr<TLog>& log);
 
     TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {});
 
 private:
-    NScheme::TSchemeClient& SchemeClient;
-    NTable::TTableClient& TableClient;
+    const TDriver& Driver;
+    std::shared_ptr<TLog> Log;
 
 }; // TDumpClient
 

ydb/public/lib/ydb_cli/dump/restore_impl.cpp

@@ -95,17 +95,12 @@ bool IsOperationStarted(TStatus operationStatus) {
 
 } // anonymous
 
-TRestoreClient::TRestoreClient(
-        TLog& log,
-        TImportClient& importClient,
-        TOperationClient& operationClient,
-        TSchemeClient& schemeClient,
-        TTableClient& tableClient)
+TRestoreClient::TRestoreClient(const TDriver& driver, TLog& log)
     : Log(log)
-    , ImportClient(importClient)
-    , OperationClient(operationClient)
-    , SchemeClient(schemeClient)
-    , TableClient(tableClient)
+    , ImportClient(driver)
+    , OperationClient(driver)
+    , SchemeClient(driver)
+    , TableClient(driver)
 {
 }
 

ydb/public/lib/ydb_cli/dump/restore_impl.h

@@ -8,6 +8,7 @@
 #include <ydb/public/sdk/cpp/client/ydb_table/table.h>
 
 #include <util/folder/path.h>
+#include <util/generic/hash_set.h>
 
 namespace NYdb {
 namespace NDump {
@@ -46,21 +47,16 @@ class TRestoreClient {
     TRestoreResult RestorePermissions(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, const THashSet<TString>& oldEntries);
 
 public:
-    explicit TRestoreClient(
-        TLog& log,
-        NImport::TImportClient& importClient,
-        NOperation::TOperationClient& operationClient,
-        NScheme::TSchemeClient& SchemeClient,
-        NTable::TTableClient& tableClient);
+    explicit TRestoreClient(const TDriver& driver, TLog& log);
 
     TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings = {});
 
 private:
     TLog& Log;
-    NImport::TImportClient& ImportClient;
-    NOperation::TOperationClient& OperationClient;
-    NScheme::TSchemeClient& SchemeClient;
-    NTable::TTableClient& TableClient;
+    NImport::TImportClient ImportClient;
+    NOperation::TOperationClient OperationClient;
+    NScheme::TSchemeClient SchemeClient;
+    NTable::TTableClient TableClient;
 
 }; // TRestoreClient
 

ydb/public/lib/ydb_cli/dump/restore_import_data.cpp

@@ -2,6 +2,7 @@
 
 #include <ydb/public/lib/ydb_cli/common/retry_func.h>
 #include <ydb/public/lib/ydb_cli/dump/util/util.h>
+
 #include <library/cpp/string_utils/quote/quote.h>
 #include <library/cpp/bucket_quoter/bucket_quoter.h>
 

ydb/public/lib/ydb_cli/dump/ya.make

@@ -11,6 +11,7 @@ SRCS(
 PEERDIR(
     library/cpp/bucket_quoter
     library/cpp/logger
+    library/cpp/regex/pcre
     library/cpp/string_utils/quote
     ydb/library/backup
     ydb/public/api/protos
@@ -19,4 +20,6 @@ PEERDIR(
     ydb/public/sdk/cpp/client/ydb_proto
 )
 
+GENERATE_ENUM_SERIALIZATION(dump.h)
+
 END()