diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index.cpp index 08f72ca6c04f..32697ebaeee1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index.cpp @@ -34,7 +34,8 @@ void TSchemeShard::Handle(TEvPrivate::TEvIndexBuildingMakeABill::TPtr& ev, const void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuildInfo::TPtr info) { Y_ABORT_UNLESS(info->BuildKind != TIndexBuildInfo::EBuildKind::BuildKindUnspecified); - db.Table().Key(info->Id).Update( + auto persistedBuildIndex = db.Table().Key(info->Id); + persistedBuildIndex.Update( NIceDb::TUpdate(info->Uid), NIceDb::TUpdate(info->DomainPathId.OwnerId), NIceDb::TUpdate(info->DomainPathId.LocalPathId), @@ -48,6 +49,17 @@ void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuil NIceDb::TUpdate(info->Limits.MaxRetries), NIceDb::TUpdate(ui32(info->BuildKind)) ); + // Persist details of the index build operation: ImplTableDescription. + // We have chosen TIndexCreationConfig's string representation as the serialization format. + { + NKikimrSchemeOp::TIndexCreationConfig serializableRepresentation; + + *serializableRepresentation.MutableIndexImplTableDescription() = info->ImplTableDescription; + + persistedBuildIndex.Update( + NIceDb::TUpdate(serializableRepresentation.SerializeAsString()) + ); + } ui32 columnNo = 0; for (ui32 i = 0; i < info->IndexColumns.size(); ++i, ++columnNo) { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 8bbdca2ac540..7e2449596802 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3092,6 +3092,14 @@ struct TIndexBuildInfo: public TSimpleRefCount { indexInfo->IndexName = row.template GetValue(); indexInfo->IndexType = row.template GetValue(); + // Restore the operation details: ImplTableDescription. + if (row.template HaveValue()) { + NKikimrSchemeOp::TIndexCreationConfig creationConfig; + Y_ABORT_UNLESS(creationConfig.ParseFromString(row.template GetValue())); + + indexInfo->ImplTableDescription = std::move(*creationConfig.MutableIndexImplTableDescription()); + } + indexInfo->State = TIndexBuildInfo::EState( row.template GetValue()); indexInfo->Issue = diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index a44c1adbef61..2d4511d15a2d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1318,6 +1318,9 @@ struct Schema : NIceDb::Schema { struct AlterMainTableTxStatus : Column<32, NScheme::NTypeIds::Uint32> { using Type = NKikimrScheme::EStatus; }; struct AlterMainTableTxDone : Column<33, NScheme::NTypeIds::Bool> {}; + // Serialized as string NKikimrSchemeOp::TIndexCreationConfig protobuf. + struct CreationConfig : Column<34, NScheme::NTypeIds::String> { using Type = TString; }; + using TKey = TableKey; using TColumns = TableColumns< Id, @@ -1352,7 +1355,8 @@ struct Schema : NIceDb::Schema { BuildKind, AlterMainTableTxId, AlterMainTableTxStatus, - AlterMainTableTxDone + AlterMainTableTxDone, + CreationConfig >; }; diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 8a415d0cad6c..fe7c69563602 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -1252,11 +1252,63 @@ TCheckFunc PartitionKeys(TVector lastShardKeys) { const auto& pathDescr = record.GetPathDescription(); UNIT_ASSERT_VALUES_EQUAL(lastShardKeys.size(), pathDescr.TablePartitionsSize()); for (size_t i = 0; i < lastShardKeys.size(); ++i) { - UNIT_ASSERT_STRING_CONTAINS(pathDescr.GetTablePartitions(i).GetEndOfRangeKeyPrefix(), lastShardKeys[i]); + const auto& partition = pathDescr.GetTablePartitions(i); + UNIT_ASSERT_STRING_CONTAINS_C( + partition.GetEndOfRangeKeyPrefix(), lastShardKeys[i], + "partition index: " << i << '\n' + << "actual key prefix: " << partition.GetEndOfRangeKeyPrefix().Quote() << '\n' + << "expected key prefix: " << lastShardKeys[i].Quote() << '\n' + ); } }; } +namespace { + +// Serializes / deserializes a value of type T to a cell vector string representation. +template +struct TSplitBoundarySerializer { + static TString Serialize(T splitBoundary) { + const auto cell = TCell::Make(splitBoundary); + TSerializedCellVec cellVec(TArrayRef(&cell, 1)); + return cellVec.ReleaseBuffer(); + } + + static TVector Deserialize(const TString& serializedCells) { + TSerializedCellVec cells(serializedCells); + TVector values; + for (const auto& cell : cells.GetCells()) { + if (cell.IsNull()) { + // the last cell + break; + } + values.emplace_back(cell.AsValue()); + } + return values; + } +}; + +} + +template +TCheckFunc SplitBoundaries(TVector&& expectedBoundaries) { + return [expectedBoundaries = std::move(expectedBoundaries)] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + const auto& pathDescr = record.GetPathDescription(); + UNIT_ASSERT_VALUES_EQUAL(pathDescr.TablePartitionsSize(), expectedBoundaries.size() + 1); + for (size_t i = 0; i < expectedBoundaries.size(); ++i) { + const auto& partition = pathDescr.GetTablePartitions(i); + const auto actualBoundary = TSplitBoundarySerializer::Deserialize(partition.GetEndOfRangeKeyPrefix()).at(0); + UNIT_ASSERT_VALUES_EQUAL_C( + actualBoundary, expectedBoundaries[i], + "partition index: " << i << '\n' + << "actual key prefix: " << partition.GetEndOfRangeKeyPrefix().Quote() << '\n' + ); + } + }; +} + +template TCheckFunc SplitBoundaries(TVector&&); + TCheckFunc ServerlessComputeResourcesMode(NKikimrSubDomains::EServerlessComputeResourcesMode serverlessComputeResourcesMode) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus()); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index 3cb8f39c5754..0f39c65f8513 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -99,6 +99,10 @@ namespace NLs { void CheckBoundaries(const NKikimrScheme::TEvDescribeSchemeResult& record); TCheckFunc PartitionCount(ui32 count); TCheckFunc PartitionKeys(TVector lastShardKeys); + // Checks if the serialized representation of an expected boundary is a prefix of the actual one. + // Similar to PartitionKeys check, but does not require you to pass split boundaries in a serialized form. + template + TCheckFunc SplitBoundaries(TVector&& expectedBoundaries); TCheckFunc FollowerCount(ui32 count); TCheckFunc CrossDataCenterFollowerCount(ui32 count); TCheckFunc AllowFollowerPromotion(bool val); diff --git a/ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp b/ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp index f24bc002567c..ef83b89235a3 100644 --- a/ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build/ut_index_build.cpp @@ -910,6 +910,78 @@ Y_UNIT_TEST_SUITE(IndexBuildTest) { } } + Y_UNIT_TEST(IndexPartitioningIsPersisted) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: [ "key" ] + )"); + env.TestWaitNotification(runtime, txId); + + Ydb::Table::GlobalIndexSettings settings; + UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(R"( + partition_at_keys { + split_points { + type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } + value { items { text_value: "alice" } } + } + split_points { + type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } + value { items { text_value: "bob" } } + } + } + partitioning_settings { + min_partitions_count: 3 + max_partitions_count: 3 + } + )", &settings)); + + TBlockEvents indexCreationBlocker(runtime, [](const auto& ev) { + const auto& modifyScheme = ev->Get()->Record.GetTransaction(0); + return modifyScheme.GetOperationType() == NKikimrSchemeOp::ESchemeOpCreateIndexBuild; + }); + + const ui64 buildIndexTx = ++txId; + TestBuildIndex(runtime, buildIndexTx, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/Table", TBuildIndexConfig{ + "Index", NKikimrSchemeOp::EIndexTypeGlobal, { "value" }, {}, + { NYdb::NTable::TGlobalIndexSettings::FromProto(settings) } + }); + + RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor()); + + indexCreationBlocker.Stop().Unblock(); + env.TestWaitNotification(runtime, buildIndexTx); + + auto buildIndexOperation = TestGetBuildIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexTx); + UNIT_ASSERT_VALUES_EQUAL_C( + (int)buildIndexOperation.GetIndexBuild().GetState(), (int)Ydb::Table::IndexBuildState::STATE_DONE, + buildIndexOperation.DebugString() + ); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + NLs::IsTable, + NLs::IndexesCount(1) + }); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Index"), { + NLs::PathExist, + NLs::IndexState(NKikimrSchemeOp::EIndexState::EIndexStateReady) + }); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Index/indexImplTable", true, true), { + NLs::IsTable, + NLs::PartitionCount(3), + NLs::MinPartitionsCountEqual(3), + NLs::MaxPartitionsCountEqual(3), + NLs::PartitionKeys({"alice", "bob", ""}) + }); + } + Y_UNIT_TEST(DropIndex) { TTestBasicRuntime runtime; TTestEnv env(runtime); diff --git a/ydb/core/tx/schemeshard/ut_index_build/ya.make b/ydb/core/tx/schemeshard/ut_index_build/ya.make index 10643c848a6a..f365a9c79f2c 100644 --- a/ydb/core/tx/schemeshard/ut_index_build/ya.make +++ b/ydb/core/tx/schemeshard/ut_index_build/ya.make @@ -14,6 +14,7 @@ PEERDIR( ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers + ydb/public/sdk/cpp/client/ydb_table ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/ut_index_build_reboots/ut_index_build_reboots.cpp b/ydb/core/tx/schemeshard/ut_index_build_reboots/ut_index_build_reboots.cpp index 8f6135105977..2f9599cbb644 100644 --- a/ydb/core/tx/schemeshard/ut_index_build_reboots/ut_index_build_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build_reboots/ut_index_build_reboots.cpp @@ -450,4 +450,75 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { }); } + + Y_UNIT_TEST(IndexPartitioning) { + TTestWithReboots t(false); + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + + TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: [ "key" ] + )"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); + } + + Ydb::Table::GlobalIndexSettings settings; + UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(R"( + partition_at_keys { + split_points { + type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } + value { items { text_value: "alice" } } + } + split_points { + type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } } + value { items { text_value: "bob" } } + } + } + partitioning_settings { + min_partitions_count: 3 + max_partitions_count: 3 + } + )", &settings)); + + const ui64 buildIndexId = ++t.TxId; + AsyncBuildIndex(runtime, buildIndexId, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/Table", TBuildIndexConfig{ + "Index", NKikimrSchemeOp::EIndexTypeGlobal, { "value" }, {}, + { NYdb::NTable::TGlobalIndexSettings::FromProto(settings) } + }); + + { + auto descr = TestGetBuildIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexId); + UNIT_ASSERT_VALUES_EQUAL((int)descr.GetIndexBuild().GetState(), (int)Ydb::Table::IndexBuildState::STATE_PREPARING); + } + + t.TestEnv->TestWaitNotification(runtime, buildIndexId); + + { + auto descr = TestGetBuildIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexId); + UNIT_ASSERT_VALUES_EQUAL((int)descr.GetIndexBuild().GetState(), (int)Ydb::Table::IndexBuildState::STATE_DONE); + } + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + NLs::IsTable, + NLs::IndexesCount(1) + }); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Index"), { + NLs::PathExist, + NLs::IndexState(NKikimrSchemeOp::EIndexState::EIndexStateReady) + }); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/Index/indexImplTable", true, true), { + NLs::IsTable, + NLs::PartitionCount(3), + NLs::MinPartitionsCountEqual(3), + NLs::MaxPartitionsCountEqual(3), + NLs::PartitionKeys({"alice", "bob", ""}) + }); + }); + } } diff --git a/ydb/core/tx/schemeshard/ut_index_build_reboots/ya.make b/ydb/core/tx/schemeshard/ut_index_build_reboots/ya.make index 594f2ceeec47..8631141f4e40 100644 --- a/ydb/core/tx/schemeshard/ut_index_build_reboots/ya.make +++ b/ydb/core/tx/schemeshard/ut_index_build_reboots/ya.make @@ -18,6 +18,7 @@ PEERDIR( ydb/core/tx ydb/core/tx/schemeshard/ut_helpers ydb/library/yql/public/udf/service/exception_policy + ydb/public/sdk/cpp/client/ydb_table ) SRCS( diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 917a11bbb1d4..a2f704ab264e 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -251,8 +251,7 @@ static void SerializeTo(const TRenameIndex& rename, Ydb::Table::RenameIndexItem& proto.set_replace_destination(rename.ReplaceDestination_); } -template -TExplicitPartitions TExplicitPartitions::FromProto(const TProto& proto) { +TExplicitPartitions TExplicitPartitions::FromProto(const Ydb::Table::ExplicitPartitions& proto) { TExplicitPartitions out; for (const auto& splitPoint : proto.split_points()) { TValue value(TType(splitPoint.type()), splitPoint.value()); @@ -2307,13 +2306,12 @@ ui64 TIndexDescription::GetSizeBytes() const { return SizeBytes; } -template -TGlobalIndexSettings TGlobalIndexSettings::FromProto(const TProto& proto) { - auto partitionsFromProto = [](const auto& proto) -> TUniformOrExplicitPartitions { +TGlobalIndexSettings TGlobalIndexSettings::FromProto(const Ydb::Table::GlobalIndexSettings& proto) { + auto partitionsFromProto = [](const Ydb::Table::GlobalIndexSettings& proto) -> TUniformOrExplicitPartitions { switch (proto.partitions_case()) { - case TProto::kUniformPartitions: + case Ydb::Table::GlobalIndexSettings::kUniformPartitions: return proto.uniform_partitions(); - case TProto::kPartitionAtKeys: + case Ydb::Table::GlobalIndexSettings::kPartitionAtKeys: return TExplicitPartitions::FromProto(proto.partition_at_keys()); default: return {}; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 1827c930457e..d150a7648b88 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -184,9 +184,7 @@ struct TExplicitPartitions { FLUENT_SETTING_VECTOR(TValue, SplitPoints); - template - static TExplicitPartitions FromProto(const TProto& proto); - + static TExplicitPartitions FromProto(const Ydb::Table::ExplicitPartitions& proto); void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const; }; @@ -196,9 +194,7 @@ struct TGlobalIndexSettings { TPartitioningSettings PartitioningSettings; TUniformOrExplicitPartitions Partitions; - template - static TGlobalIndexSettings FromProto(const TProto& proto); - + static TGlobalIndexSettings FromProto(const Ydb::Table::GlobalIndexSettings& proto); void SerializeTo(Ydb::Table::GlobalIndexSettings& proto) const; }; diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index a9352a6e571d..e1349499395f 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -4788,6 +4788,11 @@ "ColumnId": 33, "ColumnName": "AlterMainTableTxDone", "ColumnType": "Bool" + }, + { + "ColumnId": 34, + "ColumnName": "CreationConfig", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -4826,7 +4831,8 @@ 30, 31, 32, - 33 + 33, + 34 ], "RoomID": 0, "Codec": 0,