Skip to content

Commit 5dea000

Browse files
jepett0jepett0
authored
Merge 270c3df into 93e2e23 
2 parents 93e2e23 + 270c3df commit 5dea000

File tree

12 files changed

+383
-16
lines changed

12 files changed

+383
-16
lines changed

‎ydb/core/tx/schemeshard/schemeshard_build_index.cpp

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ void TSchemeShard::Handle(TEvPrivate::TEvIndexBuildingMakeABill::TPtr& ev, const
4646

4747
void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuildInfo& info) {
4848
Y_ABORT_UNLESS(info.BuildKind != TIndexBuildInfo::EBuildKind::BuildKindUnspecified);
49-
db.Table<Schema::IndexBuild>().Key(info.Id).Update(
49+
auto persistedBuildIndex = db.Table<Schema::IndexBuild>().Key(info.Id);
50+
persistedBuildIndex.Update(
5051
NIceDb::TUpdate<Schema::IndexBuild::Uid>(info.Uid),
5152
NIceDb::TUpdate<Schema::IndexBuild::DomainOwnerId>(info.DomainPathId.OwnerId),
5253
NIceDb::TUpdate<Schema::IndexBuild::DomainLocalId>(info.DomainPathId.LocalPathId),
@@ -59,9 +60,28 @@ void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuil
5960
NIceDb::TUpdate<Schema::IndexBuild::MaxShards>(info.Limits.MaxShards),
6061
NIceDb::TUpdate<Schema::IndexBuild::MaxRetries>(info.Limits.MaxRetries),
6162
NIceDb::TUpdate<Schema::IndexBuild::BuildKind>(ui32(info.BuildKind))
62-
63-
// TODO save info.ImplTableDescriptions
6463
);
64+
// Persist details of the index build operation: ImplTableDescriptions and SpecializedIndexDescription.
65+
// We have chosen TIndexCreationConfig's string representation as the serialization format.
66+
if (bool hasSpecializedDescription = !std::holds_alternative<std::monostate>(info.SpecializedIndexDescription);
67+
info.ImplTableDescriptions || hasSpecializedDescription
68+
) {
69+
NKikimrSchemeOp::TIndexCreationConfig serializableRepresentation;
70+
71+
for (const auto& description : info.ImplTableDescriptions) {
72+
*serializableRepresentation.AddIndexImplTableDescriptions() = description;
73+
}
74+
75+
std::visit([&]<typename T>(const T& specializedDescription) {
76+
if constexpr (std::is_same_v<T, NKikimrSchemeOp::TVectorIndexKmeansTreeDescription>) {
77+
*serializableRepresentation.MutableVectorIndexKmeansTreeDescription() = specializedDescription;
78+
}
79+
}, info.SpecializedIndexDescription);
80+
81+
persistedBuildIndex.Update(
82+
NIceDb::TUpdate<Schema::IndexBuild::CreationConfig>(serializableRepresentation.SerializeAsString())
83+
);
84+
}
6585

6686
ui32 columnNo = 0;
6787
for (ui32 i = 0; i < info.IndexColumns.size(); ++i, ++columnNo) {

‎ydb/core/tx/schemeshard/schemeshard_info_types.h

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2991,7 +2991,7 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
29912991
// TODO(mbkkt) move to TVectorIndexKmeansTreeDescription
29922992
ui32 K = 4;
29932993
ui32 Levels = 5;
2994-
2994+
29952995
// progress
29962996
enum EState : ui32 {
29972997
Sample = 0,
@@ -3007,7 +3007,7 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
30073007
EState State = Sample;
30083008

30093009
ui32 ChildBegin = 1; // included
3010-
3010+
30113011
static ui32 BinPow(ui32 k, ui32 l) {
30123012
ui32 r = 1;
30133013
while (l != 0) {
@@ -3282,7 +3282,26 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
32823282
indexInfo->IndexName = row.template GetValue<Schema::IndexBuild::IndexName>();
32833283
indexInfo->IndexType = row.template GetValue<Schema::IndexBuild::IndexType>();
32843284

3285-
// TODO load indexInfo->ImplTableDescriptions
3285+
// Restore the operation details: ImplTableDescriptions and SpecializedIndexDescription.
3286+
if (row.template HaveValue<Schema::IndexBuild::CreationConfig>()) {
3287+
NKikimrSchemeOp::TIndexCreationConfig creationConfig;
3288+
Y_ABORT_UNLESS(creationConfig.ParseFromString(row.template GetValue<Schema::IndexBuild::CreationConfig>()));
3289+
3290+
auto& descriptions = *creationConfig.MutableIndexImplTableDescriptions();
3291+
indexInfo->ImplTableDescriptions.reserve(descriptions.size());
3292+
for (auto& description : descriptions) {
3293+
indexInfo->ImplTableDescriptions.emplace_back(std::move(description));
3294+
}
3295+
3296+
switch (creationConfig.GetSpecializedIndexDescriptionCase()) {
3297+
case NKikimrSchemeOp::TIndexCreationConfig::kVectorIndexKmeansTreeDescription:
3298+
indexInfo->SpecializedIndexDescription = std::move(*creationConfig.MutableVectorIndexKmeansTreeDescription());
3299+
break;
3300+
case NKikimrSchemeOp::TIndexCreationConfig::SPECIALIZEDINDEXDESCRIPTION_NOT_SET:
3301+
/* do nothing */
3302+
break;
3303+
}
3304+
}
32863305

32873306
indexInfo->State = TIndexBuildInfo::EState(
32883307
row.template GetValue<Schema::IndexBuild::State>());

‎ydb/core/tx/schemeshard/schemeshard_schema.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1325,6 +1325,9 @@ struct Schema : NIceDb::Schema {
13251325
struct AlterMainTableTxStatus : Column<32, NScheme::NTypeIds::Uint32> { using Type = NKikimrScheme::EStatus; };
13261326
struct AlterMainTableTxDone : Column<33, NScheme::NTypeIds::Bool> {};
13271327

1328+
// Serialized as string NKikimrSchemeOp::TIndexCreationConfig protobuf.
1329+
struct CreationConfig : Column<34, NScheme::NTypeIds::String> { using Type = TString; };
1330+
13281331
using TKey = TableKey<Id>;
13291332
using TColumns = TableColumns<
13301333
Id,
@@ -1359,7 +1362,8 @@ struct Schema : NIceDb::Schema {
13591362
BuildKind,
13601363
AlterMainTableTxId,
13611364
AlterMainTableTxStatus,
1362-
AlterMainTableTxDone
1365+
AlterMainTableTxDone,
1366+
CreationConfig
13631367
>;
13641368
};
13651369

‎ydb/core/tx/schemeshard/ut_helpers/helpers.cpp

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1699,11 +1699,23 @@ namespace NSchemeShardUT_Private {
16991699
} break;
17001700
case NKikimrSchemeOp::EIndexTypeGlobalVectorKmeansTree: {
17011701
auto& settings = *index.mutable_global_vector_kmeans_tree_index();
1702-
settings = Ydb::Table::GlobalVectorKMeansTreeIndex();
1703-
// some random valid settings
1704-
settings.mutable_vector_settings()->mutable_settings()->set_vector_type(Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT);
1705-
settings.mutable_vector_settings()->mutable_settings()->set_vector_dimension(42);
1706-
settings.mutable_vector_settings()->mutable_settings()->set_metric(Ydb::Table::VectorIndexSettings::DISTANCE_COSINE);
1702+
1703+
auto& vectorIndexSettings = *settings.mutable_vector_settings()->mutable_settings();
1704+
if (cfg.VectorIndexSettings) {
1705+
cfg.VectorIndexSettings->SerializeTo(vectorIndexSettings);
1706+
} else {
1707+
// some random valid settings
1708+
vectorIndexSettings.set_vector_type(Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT);
1709+
vectorIndexSettings.set_vector_dimension(42);
1710+
vectorIndexSettings.set_metric(Ydb::Table::VectorIndexSettings::DISTANCE_COSINE);
1711+
}
1712+
1713+
if (cfg.GlobalIndexSettings) {
1714+
cfg.GlobalIndexSettings[0].SerializeTo(*settings.mutable_level_table_settings());
1715+
if (cfg.GlobalIndexSettings.size() > 1) {
1716+
cfg.GlobalIndexSettings[1].SerializeTo(*settings.mutable_posting_table_settings());
1717+
}
1718+
}
17071719
} break;
17081720
default:
17091721
UNIT_ASSERT_C(false, "Unknown index type: " << static_cast<ui32>(cfg.IndexType));

‎ydb/core/tx/schemeshard/ut_helpers/helpers.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464

6565
namespace NYdb::NTable {
6666
struct TGlobalIndexSettings;
67+
struct TVectorIndexSettings;
6768
}
6869

6970
namespace NSchemeShardUT_Private {
@@ -371,6 +372,8 @@ namespace NSchemeShardUT_Private {
371372
TVector<TString> IndexColumns;
372373
TVector<TString> DataColumns;
373374
TVector<NYdb::NTable::TGlobalIndexSettings> GlobalIndexSettings = {};
375+
// implementation note: it was made a pointer, not optional, to enable forward declaration
376+
std::unique_ptr<NYdb::NTable::TVectorIndexSettings> VectorIndexSettings = {};
374377
};
375378

376379
std::unique_ptr<TEvIndexBuilder::TEvCreateRequest> CreateBuildColumnRequest(ui64 id, const TString& dbName, const TString& src, const TString& columnName, const Ydb::TypedValue& literal);

‎ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -853,7 +853,7 @@ TCheckFunc IndexDataColumns(const TVector<TString>& dataColumnNames) {
853853
};
854854
}
855855

856-
TCheckFunc VectorIndexDescription(Ydb::Table::VectorIndexSettings_Metric metric,
856+
TCheckFunc VectorIndexDescription(Ydb::Table::VectorIndexSettings_Metric metric,
857857
Ydb::Table::VectorIndexSettings_VectorType vectorType,
858858
ui32 vectorDimension
859859
) {
@@ -1309,11 +1309,63 @@ TCheckFunc PartitionKeys(TVector<TString> lastShardKeys) {
13091309
const auto& pathDescr = record.GetPathDescription();
13101310
UNIT_ASSERT_VALUES_EQUAL(lastShardKeys.size(), pathDescr.TablePartitionsSize());
13111311
for (size_t i = 0; i < lastShardKeys.size(); ++i) {
1312-
UNIT_ASSERT_STRING_CONTAINS(pathDescr.GetTablePartitions(i).GetEndOfRangeKeyPrefix(), lastShardKeys[i]);
1312+
const auto& partition = pathDescr.GetTablePartitions(i);
1313+
UNIT_ASSERT_STRING_CONTAINS_C(
1314+
partition.GetEndOfRangeKeyPrefix(), lastShardKeys[i],
1315+
"partition index: " << i << '\n'
1316+
<< "actual key prefix: " << partition.GetEndOfRangeKeyPrefix().Quote() << '\n'
1317+
<< "expected key prefix: " << lastShardKeys[i].Quote() << '\n'
1318+
);
13131319
}
13141320
};
13151321
}
13161322

1323+
namespace {
1324+
1325+
// Serializes / deserializes a value of type T to a cell vector string representation.
1326+
template <typename T>
1327+
struct TSplitBoundarySerializer {
1328+
static TString Serialize(T splitBoundary) {
1329+
const auto cell = TCell::Make(splitBoundary);
1330+
TSerializedCellVec cellVec(TArrayRef<const TCell>(&cell, 1));
1331+
return cellVec.ReleaseBuffer();
1332+
}
1333+
1334+
static TVector<T> Deserialize(const TString& serializedCells) {
1335+
TSerializedCellVec cells(serializedCells);
1336+
TVector<T> values;
1337+
for (const auto& cell : cells.GetCells()) {
1338+
if (cell.IsNull()) {
1339+
// the last cell
1340+
break;
1341+
}
1342+
values.emplace_back(cell.AsValue<T>());
1343+
}
1344+
return values;
1345+
}
1346+
};
1347+
1348+
}
1349+
1350+
template <typename T>
1351+
TCheckFunc SplitBoundaries(TVector<T>&& expectedBoundaries) {
1352+
return [expectedBoundaries = std::move(expectedBoundaries)] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
1353+
const auto& pathDescr = record.GetPathDescription();
1354+
UNIT_ASSERT_VALUES_EQUAL(pathDescr.TablePartitionsSize(), expectedBoundaries.size() + 1);
1355+
for (size_t i = 0; i < expectedBoundaries.size(); ++i) {
1356+
const auto& partition = pathDescr.GetTablePartitions(i);
1357+
const auto actualBoundary = TSplitBoundarySerializer<T>::Deserialize(partition.GetEndOfRangeKeyPrefix()).at(0);
1358+
UNIT_ASSERT_VALUES_EQUAL_C(
1359+
actualBoundary, expectedBoundaries[i],
1360+
"partition index: " << i << '\n'
1361+
<< "actual key prefix: " << partition.GetEndOfRangeKeyPrefix().Quote() << '\n'
1362+
);
1363+
}
1364+
};
1365+
}
1366+
1367+
template TCheckFunc SplitBoundaries<ui32>(TVector<ui32>&&);
1368+
13171369
TCheckFunc ServerlessComputeResourcesMode(NKikimrSubDomains::EServerlessComputeResourcesMode serverlessComputeResourcesMode) {
13181370
return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
13191371
UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus());

‎ydb/core/tx/schemeshard/ut_helpers/ls_checks.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,10 @@ namespace NLs {
102102
void CheckBoundaries(const NKikimrScheme::TEvDescribeSchemeResult& record);
103103
TCheckFunc PartitionCount(ui32 count);
104104
TCheckFunc PartitionKeys(TVector<TString> lastShardKeys);
105+
// Checks if the serialized representation of an expected boundary is a prefix of the actual one.
106+
// Similar to PartitionKeys check, but does not require you to pass split boundaries in a serialized form.
107+
template <typename T>
108+
TCheckFunc SplitBoundaries(TVector<T>&& expectedBoundaries);
105109
TCheckFunc FollowerCount(ui32 count);
106110
TCheckFunc CrossDataCenterFollowerCount(ui32 count);
107111
TCheckFunc AllowFollowerPromotion(bool val);
@@ -141,7 +145,7 @@ namespace NLs {
141145
TCheckFunc IndexState(NKikimrSchemeOp::EIndexState state);
142146
TCheckFunc IndexKeys(const TVector<TString>& keyNames);
143147
TCheckFunc IndexDataColumns(const TVector<TString>& dataColumnNames);
144-
148+
145149
TCheckFunc VectorIndexDescription(Ydb::Table::VectorIndexSettings_Metric metric,
146150
Ydb::Table::VectorIndexSettings_VectorType vectorType,
147151
ui32 vectorDimension

‎ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -934,6 +934,78 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) {
934934
}
935935
}
936936

937+
Y_UNIT_TEST(IndexPartitioningIsPersisted) {
938+
TTestBasicRuntime runtime;
939+
TTestEnv env(runtime);
940+
ui64 txId = 100;
941+
942+
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
943+
Name: "Table"
944+
Columns { Name: "key" Type: "Uint64" }
945+
Columns { Name: "value" Type: "Utf8" }
946+
KeyColumnNames: [ "key" ]
947+
)");
948+
env.TestWaitNotification(runtime, txId);
949+
950+
Ydb::Table::GlobalIndexSettings settings;
951+
UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(R"(
952+
partition_at_keys {
953+
split_points {
954+
type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } }
955+
value { items { text_value: "alice" } }
956+
}
957+
split_points {
958+
type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } }
959+
value { items { text_value: "bob" } }
960+
}
961+
}
962+
partitioning_settings {
963+
min_partitions_count: 3
964+
max_partitions_count: 3
965+
}
966+
)", &settings));
967+
968+
TBlockEvents<TEvSchemeShard::TEvModifySchemeTransaction> indexCreationBlocker(runtime, [](const auto& ev) {
969+
const auto& modifyScheme = ev->Get()->Record.GetTransaction(0);
970+
return modifyScheme.GetOperationType() == NKikimrSchemeOp::ESchemeOpCreateIndexBuild;
971+
});
972+
973+
const ui64 buildIndexTx = ++txId;
974+
TestBuildIndex(runtime, buildIndexTx, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/Table", TBuildIndexConfig{
975+
"Index", NKikimrSchemeOp::EIndexTypeGlobal, { "value" }, {},
976+
{ NYdb::NTable::TGlobalIndexSettings::FromProto(settings) }
977+
});
978+
979+
RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor());
980+
981+
indexCreationBlocker.Stop().Unblock();
982+
env.TestWaitNotification(runtime, buildIndexTx);
983+
984+
auto buildIndexOperation = TestGetBuildIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexTx);
985+
UNIT_ASSERT_VALUES_EQUAL_C(
986+
buildIndexOperation.GetIndexBuild().GetState(), Ydb::Table::IndexBuildState::STATE_DONE,
987+
buildIndexOperation.DebugString()
988+
);
989+
990+
TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {
991+
NLs::IsTable,
992+
NLs::IndexesCount(1)
993+
});
994+
995+
TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Index"), {
996+
NLs::PathExist,
997+
NLs::IndexState(NKikimrSchemeOp::EIndexState::EIndexStateReady)
998+
});
999+
1000+
TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Index/indexImplTable", true, true), {
1001+
NLs::IsTable,
1002+
NLs::PartitionCount(3),
1003+
NLs::MinPartitionsCountEqual(3),
1004+
NLs::MaxPartitionsCountEqual(3),
1005+
NLs::PartitionKeys({"alice", "bob", ""})
1006+
});
1007+
}
1008+
9371009
Y_UNIT_TEST(DropIndex) {
9381010
TTestBasicRuntime runtime;
9391011
TTestEnv env(runtime);

0 commit comments

Comments
 (0)