Skip to content

Commit 7abf265

Browse files
jepett0jepett0
committed
test if my changes break other tests
1 parent 58953d8 commit 7abf265

File tree

3 files changed

+193
-24
lines changed

3 files changed

+193
-24
lines changed

‎ydb/core/testlib/test_client.cpp

+6-2
Original file line numberDiff line numberDiff line change
@@ -1669,7 +1669,7 @@ namespace Tests {
16691669

16701670
NMsgBusProxy::EResponseStatus TClient::CreateTableWithUniformShardedIndex(const TString& parent,
16711671
const NKikimrSchemeOp::TTableDescription &table, const TString& indexName, const TVector<TString> indexColumns,
1672-
NKikimrSchemeOp::EIndexType type, const TVector<TString> dataColumns, TDuration timeout)
1672+
NKikimrSchemeOp::EIndexType type, const TVector<TString> dataColumns, ui32 partitionCount, TDuration timeout)
16731673
{
16741674
TAutoPtr<NMsgBusProxy::TBusSchemeOperation> request(new NMsgBusProxy::TBusSchemeOperation());
16751675
auto *op = request->Record.MutableTransaction()->MutableModifyScheme();
@@ -1690,7 +1690,11 @@ namespace Tests {
16901690
}
16911691

16921692
indexDesc->SetType(type);
1693-
indexDesc->AddIndexImplTableDescriptions()->SetUniformPartitionsCount(16);
1693+
auto* indexImplTableDescription = indexDesc->AddIndexImplTableDescriptions();
1694+
indexImplTableDescription->SetUniformPartitionsCount(partitionCount);
1695+
auto* partitioningPolicy = indexImplTableDescription->MutablePartitionConfig()->MutablePartitioningPolicy();
1696+
partitioningPolicy->SetMinPartitionsCount(partitionCount);
1697+
partitioningPolicy->SetMaxPartitionsCount(partitionCount);
16941698
}
16951699

16961700
TAutoPtr<NBus::TBusMessage> reply;

‎ydb/core/testlib/test_client.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ namespace Tests {
438438
NMsgBusProxy::EResponseStatus CreateTableWithUniformShardedIndex(const TString& parent,
439439
const NKikimrSchemeOp::TTableDescription &table, const TString& indexName,
440440
const TVector<TString> indexColumns, NKikimrSchemeOp::EIndexType type,
441-
const TVector<TString> dataColumns = {}, TDuration timeout = TDuration::Seconds(5000));
441+
const TVector<TString> dataColumns = {}, ui32 partitionCount = 16, TDuration timeout = TDuration::Seconds(5000));
442442
NMsgBusProxy::EResponseStatus SplitTable(const TString& table, ui64 datashardId, ui64 border, TDuration timeout = TDuration::Seconds(5000));
443443
NMsgBusProxy::EResponseStatus CopyTable(const TString& parent, const TString& name, const TString& src);
444444
NMsgBusProxy::EResponseStatus CreateKesus(const TString& parent, const TString& name);

‎ydb/services/ydb/backup_ut/ydb_backup_ut.cpp

+186-21
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include "ydb_common_ut.h"
22

3+
#include <ydb/core/tx/datashard/datashard.h>
34
#include <ydb/core/wrappers/ut_helpers/s3_mock.h>
45

56
#include <ydb/public/lib/ydb_cli/dump/dump.h>
@@ -13,6 +14,7 @@
1314
#include <ydb/library/backup/backup.h>
1415

1516
#include <library/cpp/testing/hook/hook.h>
17+
#include <library/cpp/testing/unittest/registar.h>
1618

1719
#include <aws/core/Aws.h>
1820

@@ -21,6 +23,8 @@ using namespace NYdb::NTable;
2123

2224
namespace {
2325

26+
#define DEBUG_HINT (TStringBuilder() << "at line " << __LINE__)
27+
2428
void ExecuteDataDefinitionQuery(TSession& session, const TString& script) {
2529
const auto result = session.ExecuteSchemeQuery(script).ExtractValueSync();
2630
UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString());
@@ -50,15 +54,19 @@ ui64 GetUint64(const TValue& value) {
5054
return TValueParser(value).GetUint64();
5155
}
5256

53-
auto CreateMinPartitionsChecker(ui64 expectedMinPartitions) {
57+
auto CreateMinPartitionsChecker(ui64 expectedMinPartitions, const TString& debugHint = "") {
5458
return [=](const TTableDescription& tableDescription) {
55-
return tableDescription.GetPartitioningSettings().GetMinPartitionsCount() == expectedMinPartitions;
59+
UNIT_ASSERT_VALUES_EQUAL_C(
60+
tableDescription.GetPartitioningSettings().GetMinPartitionsCount(),
61+
expectedMinPartitions,
62+
debugHint
63+
);
5664
};
5765
}
5866

59-
auto CreatePartitionCountChecker(ui64 expectedPartitionCount) {
67+
auto CreatePartitionCountChecker(ui64 expectedPartitionCount, const TString& debugHint = "") {
6068
return [=](const TTableDescription& tableDescription) {
61-
return tableDescription.GetPartitionsCount() == expectedPartitionCount;
69+
UNIT_ASSERT_VALUES_EQUAL_C(tableDescription.GetPartitionsCount(), expectedPartitionCount, debugHint);
6270
};
6371
}
6472

@@ -68,13 +76,7 @@ void CheckTableDescription(TSession& session, const TString& path, auto&& checke
6876
auto describeResult = session.DescribeTable(path, settings).ExtractValueSync();
6977
UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString());
7078
auto tableDescription = describeResult.GetTableDescription();
71-
Ydb::Table::CreateTableRequest descriptionProto;
72-
// The purpose of translating to CreateTableRequest is solely to produce a clearer error message.
73-
tableDescription.SerializeTo(descriptionProto);
74-
UNIT_ASSERT_C(
75-
checker(tableDescription),
76-
descriptionProto.DebugString()
77-
);
79+
checker(tableDescription);
7880
}
7981

8082
}
@@ -175,7 +177,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
175177
table, minPartitions
176178
));
177179

178-
CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions));
180+
CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT));
179181

180182
TTempDir tempDir;
181183
const auto& pathToBackup = tempDir.Path();
@@ -190,7 +192,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
190192
)", table
191193
));
192194
Restore(backupClient, pathToBackup, "/Root");
193-
CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions));
195+
CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT));
194196
}
195197

196198
Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) {
@@ -221,7 +223,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
221223
)", table, index, minPartitions
222224
));
223225

224-
CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions));
226+
CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT));
225227

226228
TTempDir tempDir;
227229
const auto& pathToBackup = tempDir.Path();
@@ -236,7 +238,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
236238
)", table
237239
));
238240
Restore(backupClient, pathToBackup, "/Root");
239-
CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions));
241+
CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT));
240242
}
241243

242244
}
@@ -295,6 +297,10 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
295297
ui16 GetS3Port() const {
296298
return s3Port;
297299
}
300+
301+
const THashMap<TString, TString>& GetS3Data() const {
302+
return s3Mock.GetData();
303+
}
298304
};
299305

300306
template <typename TOperation>
@@ -426,7 +432,7 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
426432
table, minPartitions
427433
));
428434

429-
CheckTableDescription(testEnv.GetSession(), table, CreateMinPartitionsChecker(minPartitions));
435+
CheckTableDescription(testEnv.GetSession(), table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT));
430436

431437
NExport::TExportClient exportClient(testEnv.GetDriver());
432438
NImport::TImportClient importClient(testEnv.GetDriver());
@@ -441,7 +447,7 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
441447
));
442448

443449
ImportFromS3(importClient, testEnv.GetS3Port(), operationClient, "table", table);
444-
CheckTableDescription(testEnv.GetSession(), table, CreateMinPartitionsChecker(minPartitions));
450+
CheckTableDescription(testEnv.GetSession(), table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT));
445451
}
446452

447453
Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) {
@@ -469,7 +475,7 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
469475
)", table, index, minPartitions
470476
));
471477

472-
CheckTableDescription(testEnv.GetSession(), indexTablePath, CreateMinPartitionsChecker(minPartitions));
478+
CheckTableDescription(testEnv.GetSession(), indexTablePath, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT));
473479

474480
NExport::TExportClient exportClient(testEnv.GetDriver());
475481
NImport::TImportClient importClient(testEnv.GetDriver());
@@ -484,7 +490,7 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
484490
));
485491

486492
ImportFromS3(importClient, testEnv.GetS3Port(), operationClient, "table", table);
487-
CheckTableDescription(testEnv.GetSession(), indexTablePath, CreateMinPartitionsChecker(minPartitions));
493+
CheckTableDescription(testEnv.GetSession(), indexTablePath, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT));
488494
}
489495

490496
Y_UNIT_TEST(RestoreTableSplitBoundaries) {
@@ -505,7 +511,7 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
505511
table, partitions
506512
));
507513

508-
CheckTableDescription(testEnv.GetSession(), table, CreatePartitionCountChecker(partitions),
514+
CheckTableDescription(testEnv.GetSession(), table, CreatePartitionCountChecker(partitions, DEBUG_HINT),
509515
TDescribeTableSettings().WithTableStatistics(true)
510516
);
511517

@@ -522,7 +528,166 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
522528
));
523529

524530
ImportFromS3(importClient, testEnv.GetS3Port(), operationClient, "table", table);
525-
CheckTableDescription(testEnv.GetSession(), table, CreatePartitionCountChecker(partitions),
531+
CheckTableDescription(testEnv.GetSession(), table, CreatePartitionCountChecker(partitions, DEBUG_HINT),
532+
TDescribeTableSettings().WithTableStatistics(true)
533+
);
534+
}
535+
536+
Y_UNIT_TEST(RestoreIndexTableSplitBoundaries) {
537+
TS3TestEnv testEnv;
538+
539+
NDataShard::gDbStatsReportInterval = TDuration::Zero();
540+
NDataShard::gDbStatsDataSizeResolution = 1; // to do: try to remove
541+
NDataShard::gDbStatsRowCountResolution = 1; // to do: try to remove
542+
543+
constexpr const char* table = "/Root/table";
544+
constexpr const char* index = "byValue";
545+
const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable");
546+
constexpr int partitions = 5;
547+
548+
constexpr const char* textDescription = R"(
549+
Name: "table"
550+
Columns { Name: "Key" Type: "Uint32"}
551+
Columns { Name: "Value" Type: "Uint32"}
552+
Columns { Name: "Payload" Type: "Utf8"}
553+
KeyColumnNames: ["Key"]
554+
)";
555+
NKikimrSchemeOp::TTableDescription protoDescription;
556+
UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(textDescription, &protoDescription));
557+
TClient client(*testEnv.GetServer().ServerSettings);
558+
// const auto status = client.CreateTable("/Root", tableDescr);
559+
const auto status = client.CreateTableWithUniformShardedIndex(
560+
"/Root", protoDescription, index, {"Value"}, NKikimrSchemeOp::EIndexTypeGlobal, {"Payload"}, partitions
561+
);
562+
UNIT_ASSERT_VALUES_EQUAL(status, NMsgBusProxy::EResponseStatus::MSTATUS_OK);
563+
/*
564+
ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"(
565+
CREATE TABLE `%s` (
566+
Key Uint32,
567+
Value Utf8,
568+
PRIMARY KEY (Key),
569+
INDEX %s GLOBAL ON (Value)
570+
);
571+
)",
572+
table, index
573+
));
574+
575+
ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"(
576+
ALTER TABLE `%s` ALTER INDEX %s SET (
577+
AUTO_PARTITIONING_PARTITION_SIZE_MB = 1,
578+
AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d,
579+
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = %d
580+
);
581+
)", table, index, partitions, partitions
582+
));
583+
const auto oneMegabyteValue = TString(100'000, 'a');
584+
ExecuteDataModificationQuery(testEnv.GetSession(), Sprintf(R"(
585+
UPSERT INTO `%s` (
586+
Key,
587+
Value
588+
)
589+
VALUES
590+
(1, "%s"),
591+
(2, "%s"),
592+
(3, "%s"),
593+
(4, "%s"),
594+
(5, "%s"),
595+
(6, "%s"),
596+
(7, "%s"),
597+
(8, "%s"),
598+
(9, "%s"),
599+
(10, "%s"),
600+
(11, "%s"),
601+
(12, "%s"),
602+
(13, "%s"),
603+
(14, "%s"),
604+
(15, "%s"),
605+
(16, "%s"),
606+
(17, "%s"),
607+
(18, "%s"),
608+
(19, "%s"),
609+
(20, "%s"),
610+
(21, "%s"),
611+
(22, "%s"),
612+
(23, "%s"),
613+
(24, "%s"),
614+
(25, "%s"),
615+
(26, "%s"),
616+
(27, "%s"),
617+
(28, "%s"),
618+
(29, "%s"),
619+
(30, "%s"),
620+
(31, "%s"),
621+
(32, "%s"),
622+
(33, "%s");
623+
)",
624+
table,
625+
oneMegabyteValue.c_str(),
626+
oneMegabyteValue.c_str(),
627+
oneMegabyteValue.c_str(),
628+
oneMegabyteValue.c_str(),
629+
oneMegabyteValue.c_str(),
630+
oneMegabyteValue.c_str(),
631+
oneMegabyteValue.c_str(),
632+
oneMegabyteValue.c_str(),
633+
oneMegabyteValue.c_str(),
634+
oneMegabyteValue.c_str(),
635+
oneMegabyteValue.c_str(),
636+
oneMegabyteValue.c_str(),
637+
oneMegabyteValue.c_str(),
638+
oneMegabyteValue.c_str(),
639+
oneMegabyteValue.c_str(),
640+
oneMegabyteValue.c_str(),
641+
oneMegabyteValue.c_str(),
642+
oneMegabyteValue.c_str(),
643+
oneMegabyteValue.c_str(),
644+
oneMegabyteValue.c_str(),
645+
oneMegabyteValue.c_str(),
646+
oneMegabyteValue.c_str(),
647+
oneMegabyteValue.c_str(),
648+
oneMegabyteValue.c_str(),
649+
oneMegabyteValue.c_str(),
650+
oneMegabyteValue.c_str(),
651+
oneMegabyteValue.c_str(),
652+
oneMegabyteValue.c_str(),
653+
oneMegabyteValue.c_str(),
654+
oneMegabyteValue.c_str(),
655+
oneMegabyteValue.c_str(),
656+
oneMegabyteValue.c_str(),
657+
oneMegabyteValue.c_str()
658+
));
659+
Sleep(TDuration::Seconds(60));
660+
*/
661+
662+
CheckTableDescription(testEnv.GetSession(), indexTablePath, CreatePartitionCountChecker(partitions, DEBUG_HINT),
663+
TDescribeTableSettings().WithTableStatistics(true)
664+
);
665+
666+
NExport::TExportClient exportClient(testEnv.GetDriver());
667+
NImport::TImportClient importClient(testEnv.GetDriver());
668+
NOperation::TOperationClient operationClient(testEnv.GetDriver());
669+
670+
ExportToS3(exportClient, testEnv.GetS3Port(), operationClient, table, "table");
671+
// debug begin
672+
const auto& exportedData = testEnv.GetS3Data();
673+
const auto* scheme = exportedData.FindPtr("/test_bucket/table/scheme.pb");
674+
if (!scheme) {
675+
for (const auto& [name, content] : exportedData) {
676+
Cerr << "--- name: " << name << ", content: " << content << '\n';
677+
}
678+
} else {
679+
Cerr << "--- scheme:\n" << *scheme << '\n';
680+
}
681+
// debug end
682+
683+
// The table needs to be dropped before importing from S3 can proceed successfully.
684+
ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"(
685+
DROP TABLE `%s`;
686+
)", table
687+
));
688+
689+
ImportFromS3(importClient, testEnv.GetS3Port(), operationClient, "table", table);
690+
CheckTableDescription(testEnv.GetSession(), indexTablePath, CreatePartitionCountChecker(partitions, DEBUG_HINT),
526691
TDescribeTableSettings().WithTableStatistics(true)
527692
);
528693
}

0 commit comments

Comments
 (0)