From 50e31aa54204dd37c3040d93b03e59bb5e00fd1b Mon Sep 17 00:00:00 2001 From: kungasc Date: Mon, 14 Apr 2025 12:31:40 +0300 Subject: [PATCH 01/14] test 3 levels --- ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp index 248c9ebb9b4b..71dab4250232 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp @@ -2716,6 +2716,50 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { DoPositiveQueriesVectorIndexOrderByCosine(session); } + Y_UNIT_TEST(SimpleVectorIndexOrderByCosineDistanceNotNullableLevel3) { + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableVectorIndex(true); + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetFeatureFlags(featureFlags) + .SetKqpSettings({setting}); + + TKikimrRunner kikimr(serverSettings); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); + auto session = DoCreateTableForVectorIndex(db, false); + { + const TString createIndex(Q_(R"( + ALTER TABLE `/Root/TestTable` + ADD INDEX index + GLOBAL USING vector_kmeans_tree + ON (emb) + WITH (distance=cosine, vector_type="uint8", vector_dimension=2, levels=3, clusters=2); + )")); + + auto result = session.ExecuteSchemeQuery(createIndex) + .ExtractValueSync(); + + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + { + auto result = session.DescribeTable("/Root/TestTable").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); + const auto& indexes = result.GetTableDescription().GetIndexDescriptions(); + UNIT_ASSERT_EQUAL(indexes.size(), 1); + UNIT_ASSERT_EQUAL(indexes[0].GetIndexName(), "index"); + UNIT_ASSERT_EQUAL(indexes[0].GetIndexColumns(), std::vector{"emb"}); + const auto& settings = std::get(indexes[0].GetIndexSettings()); + UNIT_ASSERT_EQUAL(settings.Settings.Metric, NYdb::NTable::TVectorIndexSettings::EMetric::CosineDistance); + UNIT_ASSERT_EQUAL(settings.Settings.VectorType, NYdb::NTable::TVectorIndexSettings::EVectorType::Uint8); + UNIT_ASSERT_EQUAL(settings.Settings.VectorDimension, 2); + UNIT_ASSERT_EQUAL(settings.Levels, 3); + UNIT_ASSERT_EQUAL(settings.Clusters, 2); + } + // TODO: fix somehow? + // DoPositiveQueriesVectorIndexOrderByCosine(session); + } + void DoPositiveQueriesPrefixedVectorIndexOrderBy( TSession& session, std::string_view function, From daf1ddda5f240c146f089ddb0e3bd510a44a72bb Mon Sep 17 00:00:00 2001 From: kungasc Date: Mon, 14 Apr 2025 12:40:42 +0300 Subject: [PATCH 02/14] no name --- .../schemeshard_build_index__progress.cpp | 83 +++++-------------- 1 file changed, 23 insertions(+), 60 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index 7e2df90002e8..5942db7020d5 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -22,45 +22,6 @@ namespace NKikimr { namespace NSchemeShard { -static constexpr const char* Name(TIndexBuildInfo::EState state) noexcept { - switch (state) { - case TIndexBuildInfo::EState::Invalid: - return "Invalid"; - case TIndexBuildInfo::EState::AlterMainTable: - return "AlterMainTable"; - case TIndexBuildInfo::EState::Locking: - return "Locking"; - case TIndexBuildInfo::EState::GatheringStatistics: - return "GatheringStatistics"; - case TIndexBuildInfo::EState::Initiating: - return "Initiating"; - case TIndexBuildInfo::EState::Filling: - return "Filling"; - case TIndexBuildInfo::EState::DropBuild: - return "DropBuild"; - case TIndexBuildInfo::EState::CreateBuild: - return "CreateBuild"; - case TIndexBuildInfo::EState::Applying: - return "Applying"; - case TIndexBuildInfo::EState::Unlocking: - return "Unlocking"; - case TIndexBuildInfo::EState::Done: - return "Done"; - case TIndexBuildInfo::EState::Cancellation_Applying: - return "Cancellation_Applying"; - case TIndexBuildInfo::EState::Cancellation_Unlocking: - return "Cancellation_Unlocking"; - case TIndexBuildInfo::EState::Cancelled: - return "Cancelled"; - case TIndexBuildInfo::EState::Rejection_Applying: - return "Rejection_Applying"; - case TIndexBuildInfo::EState::Rejection_Unlocking: - return "Rejection_Unlocking"; - case TIndexBuildInfo::EState::Rejected: - return "Rejected"; - } -} - // return count, parts, step static std::tuple ComputeKMeansBoundaries(const NSchemeShard::TTableInfo& tableInfo, const TIndexBuildInfo& buildInfo) { const auto& kmeans = buildInfo.KMeans; @@ -1111,8 +1072,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Y_ABORT_UNLESS(buildInfoPtr); auto& buildInfo = *buildInfoPtr->Get(); - LOG_I("TTxBuildProgress: Resume: id# " << BuildId); - LOG_D("TTxBuildProgress: Resume: " << buildInfo); + LOG_I("TTxBuildProgress: Execute: " << BuildId << " " << buildInfo.State << " " << buildInfo); switch (buildInfo.State) { case TIndexBuildInfo::EState::Invalid: @@ -1340,13 +1300,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Y_ASSERT(buildInfo.InProgressShards.empty()); Y_ASSERT(buildInfo.DoneShards.empty()); - TTableInfo::TPtr table; - if (buildInfo.KMeans.Level == 1) { - table = Self->Tables.at(buildInfo.TablePathId); - } else { - auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); - table = Self->Tables.at(path.Dive(buildInfo.KMeans.ReadFrom())->PathId); - } + TTableInfo::TPtr table = GetScanningTable(buildInfo); auto tableColumns = NTableIndex::ExtractInfo(table); // skip dropped columns TSerializedTableRange shardRange = InfiniteRange(tableColumns.Keys.size()); static constexpr std::string_view LogPrefix = ""; @@ -1368,6 +1322,15 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil } } + TTableInfo::TPtr GetScanningTable(TIndexBuildInfo& buildInfo) { + if (buildInfo.KMeans.Level == 1) { + return Self->Tables.at(buildInfo.TablePathId); + } else { + auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); + return Self->Tables.at(path.Dive(buildInfo.KMeans.ReadFrom())->PathId); + } + } + void DoComplete(const TActorContext& ctx) override { for (auto& x: ToTabletSend) { Self->IndexBuildPipes.Create(BuildId, std::get<0>(x), std::move(std::get<2>(x)), ctx); @@ -1486,7 +1449,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyRetry: public TSchemeShard::TIndexBu case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -1640,7 +1603,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplySampleK: public TSchemeShard::TIndex case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -1765,7 +1728,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyLocalKMeans: public TSchemeShard::TI case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -1889,7 +1852,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyReshuffleKMeans: public TSchemeShard case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -2014,7 +1977,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyPrefixKMeans: public TSchemeShard::T case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -2096,7 +2059,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyUpload: public TSchemeShard::TIndexB case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -2249,7 +2212,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyProgress: public TSchemeShard::TInde case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -2351,7 +2314,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyCompleted: public TSchemeShard::TInd case TIndexBuildInfo::EState::Done: case TIndexBuildInfo::EState::Cancelled: case TIndexBuildInfo::EState::Rejected: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); } Progress(buildId); @@ -2432,7 +2395,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB if (statusCode != Ydb::StatusIds::SUCCESS) { buildInfo.Issue += TStringBuilder() - << "At " << Name(state) << " state got unsuccess propose result" + << "At " << state << " state got unsuccess propose result" << ", status: " << NKikimrScheme::EStatus_Name(record.GetStatus()) << ", reason: " << record.GetReason(); Self->PersistBuildIndexIssue(db, buildInfo); @@ -2451,7 +2414,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB // no op } else { buildInfo.Issue += TStringBuilder() - << "At " << Name(state) << " state got unsuccess propose result" + << "At " << state << " state got unsuccess propose result" << ", status: " << NKikimrScheme::EStatus_Name(record.GetStatus()) << ", reason: " << record.GetReason(); Self->PersistBuildIndexIssue(db, buildInfo); @@ -2549,7 +2512,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB case TIndexBuildInfo::EState::Done: case TIndexBuildInfo::EState::Cancelled: case TIndexBuildInfo::EState::Rejected: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); } Progress(buildId); @@ -2628,7 +2591,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyAllocate: public TSchemeShard::TInde case TIndexBuildInfo::EState::Done: case TIndexBuildInfo::EState::Cancelled: case TIndexBuildInfo::EState::Rejected: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); } Progress(buildId); From fe47c9a77ebf87522b6ab0aa5579c463b3775ad9 Mon Sep 17 00:00:00 2001 From: kungasc Date: Mon, 14 Apr 2025 13:51:58 +0300 Subject: [PATCH 03/14] add lock build state --- .../schemeshard_build_index__progress.cpp | 52 +++++++++++++++---- .../schemeshard_build_index_tx_base.cpp | 1 + .../tx/schemeshard/schemeshard_info_types.h | 1 + 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index 5942db7020d5..f6d67ebee0e0 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -1188,6 +1188,31 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Progress(BuildId); } break; + case TIndexBuildInfo::EState::LockBuild: + Y_ASSERT(buildInfo.IsBuildVectorIndex()); + if (buildInfo.ApplyTxId == InvalidTxId) { + Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { + Send(Self->SelfId(), CreateBuildPropose(Self, buildInfo), 0, ui64(BuildId)); + } else if (!buildInfo.ApplyTxDone) { + Send(Self->SelfId(), MakeHolder(ui64(buildInfo.ApplyTxId))); + } else { + buildInfo.SnapshotTxId = {}; + buildInfo.SnapshotStep = {}; + + buildInfo.ApplyTxId = {}; + buildInfo.ApplyTxStatus = NKikimrScheme::StatusSuccess; + buildInfo.ApplyTxDone = false; + + NIceDb::TNiceDb db(txc.DB); + Self->PersistBuildIndexApplyTxId(db, buildInfo); + Self->PersistBuildIndexApplyTxStatus(db, buildInfo); + Self->PersistBuildIndexApplyTxDone(db, buildInfo); + + ChangeState(BuildId, TIndexBuildInfo::EState::Filling); + Progress(BuildId); + } + break; case TIndexBuildInfo::EState::Applying: if (buildInfo.ApplyTxId == InvalidTxId) { Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); @@ -1300,7 +1325,13 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Y_ASSERT(buildInfo.InProgressShards.empty()); Y_ASSERT(buildInfo.DoneShards.empty()); - TTableInfo::TPtr table = GetScanningTable(buildInfo); + TTableInfo::TPtr table; + if (buildInfo.KMeans.Level == 1) { + table = Self->Tables.at(buildInfo.TablePathId); + } else { + auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); + table = Self->Tables.at(path.Dive(buildInfo.KMeans.ReadFrom())->PathId); + } auto tableColumns = NTableIndex::ExtractInfo(table); // skip dropped columns TSerializedTableRange shardRange = InfiniteRange(tableColumns.Keys.size()); static constexpr std::string_view LogPrefix = ""; @@ -1322,15 +1353,6 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil } } - TTableInfo::TPtr GetScanningTable(TIndexBuildInfo& buildInfo) { - if (buildInfo.KMeans.Level == 1) { - return Self->Tables.at(buildInfo.TablePathId); - } else { - auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); - return Self->Tables.at(path.Dive(buildInfo.KMeans.ReadFrom())->PathId); - } - } - void DoComplete(const TActorContext& ctx) override { for (auto& x: ToTabletSend) { Self->IndexBuildPipes.Create(BuildId, std::get<0>(x), std::move(std::get<2>(x)), ctx); @@ -1446,6 +1468,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyRetry: public TSchemeShard::TIndexBu case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: @@ -1600,6 +1623,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplySampleK: public TSchemeShard::TIndex case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: @@ -1725,6 +1749,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyLocalKMeans: public TSchemeShard::TI case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: @@ -1849,6 +1874,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyReshuffleKMeans: public TSchemeShard case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: @@ -1974,6 +2000,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyPrefixKMeans: public TSchemeShard::T case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: @@ -2056,6 +2083,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyUpload: public TSchemeShard::TIndexB case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: @@ -2209,6 +2237,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyProgress: public TSchemeShard::TInde case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: @@ -2288,6 +2317,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyCompleted: public TSchemeShard::TInd } case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Rejection_Applying: @@ -2455,6 +2485,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB } case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Rejection_Applying: { @@ -2569,6 +2600,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyAllocate: public TSchemeShard::TInde break; case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Rejection_Applying: diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp index e07d231c38e3..ecda710d1ea7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp @@ -215,6 +215,7 @@ void TSchemeShard::TIndexBuilder::TTxBase::Fill(NKikimrIndexBuilder::TIndexBuild case TIndexBuildInfo::EState::Filling: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: index.SetState(Ydb::Table::IndexBuildState::STATE_TRANSFERING_DATA); index.SetProgress(indexInfo.CalcProgressPercent()); break; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 50f489a1e902..d6b90d7e3bc5 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3040,6 +3040,7 @@ struct TIndexBuildInfo: public TSimpleRefCount { Filling = 40, DropBuild = 45, CreateBuild = 46, + LockBuild = 47, Applying = 50, Unlocking = 60, Done = 200, From 39503edfcf30d61d17941a2313ddc5b44a2ee1e8 Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 15 Apr 2025 17:50:50 +0300 Subject: [PATCH 04/14] lock impl tables --- ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp | 114 ++++++++++++++++++ ...hemeshard__operation_apply_build_index.cpp | 9 +- .../schemeshard__operation_create_lock.cpp | 31 +++-- .../schemeshard__operation_drop_lock.cpp | 2 +- .../schemeshard_build_index__progress.cpp | 106 ++++++++++------ .../schemeshard_build_index_tx_base.cpp | 5 + .../schemeshard_build_index_tx_base.h | 1 + ydb/core/tx/schemeshard/schemeshard_path.cpp | 4 + ydb/core/tx/schemeshard/schemeshard_path.h | 1 + 9 files changed, 222 insertions(+), 51 deletions(-) diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp index 71dab4250232..bcddfee4385d 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp @@ -119,6 +119,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto& server = kikimr.GetTestServer(); auto gateway = GetIcGateway(server); @@ -275,6 +277,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto& server = kikimr.GetTestServer(); auto gateway = GetIcGateway(server); @@ -389,6 +393,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -447,6 +453,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -527,6 +535,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -607,6 +617,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -792,6 +804,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -915,6 +929,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1047,6 +1063,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1282,6 +1300,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({ setting }); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1377,6 +1397,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1514,6 +1536,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1707,6 +1731,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1797,6 +1823,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1863,6 +1891,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); TScriptingClient client(kikimr.GetDriver()); { @@ -1904,6 +1934,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -2382,6 +2414,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, false); { @@ -2425,6 +2459,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, false); { @@ -2468,6 +2504,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, true); { @@ -2511,6 +2549,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, true); { @@ -2554,6 +2594,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, false); { @@ -2597,6 +2639,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, false); { @@ -2640,6 +2684,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, true); { @@ -2683,6 +2729,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, true); { @@ -2726,6 +2774,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, false); { @@ -2932,6 +2982,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, false); @@ -2977,6 +3029,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, false); @@ -3022,6 +3076,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, true); @@ -3067,6 +3123,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, true); @@ -3112,6 +3170,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, false); @@ -3249,6 +3309,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, false); @@ -3294,6 +3356,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, true); @@ -3339,6 +3403,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, true); @@ -3452,6 +3518,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3529,6 +3597,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3680,6 +3750,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetAppConfig(app); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3758,6 +3830,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3806,6 +3880,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3870,6 +3946,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -4061,6 +4139,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -4188,6 +4268,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(app); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -4528,6 +4610,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(app); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -4804,6 +4888,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5042,6 +5128,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5204,6 +5292,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5291,6 +5381,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto scheme = kikimr.GetSchemeClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5341,6 +5433,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5381,6 +5475,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5468,6 +5564,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5666,6 +5764,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda serverSettings.SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTablesWithIndex(session); @@ -5742,6 +5842,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTablesWithIndex(session); @@ -5819,6 +5921,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -6009,6 +6113,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + TScriptingClient client(kikimr.GetDriver()); auto scriptResult = client.ExecuteYqlScript(R"( @@ -6102,6 +6208,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -6159,6 +6267,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -6210,6 +6320,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -6756,6 +6868,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp index 06a5c1fa3e66..08923fa797bc 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp @@ -30,7 +30,7 @@ ISubOperation::TPtr FinalizeIndexImplTable(TOperationContext& context, const TPa return CreateFinalizeBuildIndexImplTable(partId, transaction); } -ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& nextId, const TOperationId& partId, const TString& name, const TPathId& pathId, bool& rejected) { +ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& nextId, const TOperationId& partId, const TString& name, const TPathId& pathId, const NKikimrSchemeOp::TLockGuard& lockGuard, bool& rejected) { TPath implTable = index.Child(name); Y_ABORT_UNLESS(implTable->PathId == pathId); Y_ABORT_UNLESS(implTable.LeafName() == name); @@ -48,6 +48,9 @@ ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& n } rejected = false; auto transaction = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable); + if (implTable.IsLocked()) { // some impl tables may be not locked + *transaction.MutableLockGuard() = lockGuard; + } auto operation = transaction.MutableDrop(); operation->SetName(name); return CreateDropTable(partId, transaction); @@ -98,7 +101,7 @@ TVector ApplyBuildIndex(TOperationId nextId, const TTxTrans const auto partId = NextPartId(nextId, result); if (NTableIndex::IsBuildImplTable(indexImplTableName)) { bool rejected = false; - auto op = DropIndexImplTable(index, nextId, partId, indexImplTableName, indexChildItems.second, rejected); + auto op = DropIndexImplTable(index, nextId, partId, indexImplTableName, indexChildItems.second, tx.GetLockGuard(), rejected); if (rejected) { return {std::move(op)}; } @@ -153,7 +156,7 @@ TVector CancelBuildIndex(TOperationId nextId, const TTxTran for (auto& indexChildItems : index.Base()->GetChildren()) { const auto partId = NextPartId(nextId, result); bool rejected = false; - auto op = DropIndexImplTable(index, nextId, partId, indexChildItems.first, indexChildItems.second, rejected); + auto op = DropIndexImplTable(index, nextId, partId, indexChildItems.first, indexChildItems.second, tx.GetLockGuard(), rejected); if (rejected) { return {std::move(op)}; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp index e8c02b30c81d..d78d4271c57a 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp @@ -95,6 +95,9 @@ class TCreateLock: public TSubOperation { THolder Propose(const TString&, TOperationContext& context) override { const auto& workingDir = Transaction.GetWorkingDir(); const auto& op = Transaction.GetLockConfig(); + const auto lockTxId = Transaction.HasLockGuard() && Transaction.GetLockGuard().HasOwnerTxId() + ? TTxId(Transaction.GetLockGuard().GetOwnerTxId()) + : OperationId.GetTxId(); LOG_N("TCreateLock Propose" << ": opId# " << OperationId @@ -158,17 +161,19 @@ class TCreateLock: public TSubOperation { const auto pathId = tablePath.Base()->PathId; result->SetPathId(pathId.LocalPathId); - if (tablePath.LockedBy() == OperationId.GetTxId()) { - result->SetError(NKikimrScheme::StatusAlreadyExists, TStringBuilder() << "path checks failed" - << ", path already locked by this operation" - << ", path: " << tablePath.PathString()); - return result; - } - - TString errStr; - if (!context.SS->CheckLocks(pathId, Transaction, errStr)) { - result->SetError(NKikimrScheme::StatusMultipleModifications, errStr); - return result; + if (auto lockedBy = tablePath.LockedBy(); lockedBy != InvalidTxId) { + if (lockedBy == lockTxId) { + result->SetError(NKikimrScheme::StatusAlreadyExists, TStringBuilder() << "path checks failed" + << ", path already locked by this operation" + << ", path: " << tablePath.PathString()); + return result; + } else { + result->SetError(NKikimrScheme::StatusMultipleModifications, TStringBuilder() << "path checks failed" + << ", path already locked by another operation" + << ", path: " << tablePath.PathString() + << ", locked by: " << lockedBy); + return result; + } } auto guard = context.DbGuard(); @@ -177,7 +182,7 @@ class TCreateLock: public TSubOperation { context.MemChanges.GrabNewTxState(context.SS, OperationId); context.DbChanges.PersistPath(pathId); - context.DbChanges.PersistLongLock(pathId, OperationId.GetTxId()); + context.DbChanges.PersistLongLock(pathId, lockTxId); context.DbChanges.PersistTxState(OperationId); Y_ABORT_UNLESS(!context.SS->FindTx(OperationId)); @@ -194,7 +199,7 @@ class TCreateLock: public TSubOperation { context.OnComplete.Dependence(splitOpId.GetTxId(), OperationId.GetTxId()); } - context.SS->LockedPaths[pathId] = OperationId.GetTxId(); + context.SS->LockedPaths[pathId] = lockTxId; context.SS->TabletCounters->Simple()[COUNTER_LOCKS_COUNT].Add(1); context.OnComplete.ActivateTx(OperationId); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp index 8fdfeb0a7d09..4e3cc7c79314 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp @@ -164,7 +164,7 @@ class TDropLock: public TSubOperation { const auto pathId = dstPath.Base()->PathId; result->SetPathId(pathId.LocalPathId); - if (!dstPath.LockedBy()) { + if (!dstPath.IsLocked()) { result->SetError(TEvSchemeShard::EStatus::StatusAlreadyExists, TStringBuilder() << "path checks failed" << ", path already unlocked" << ", path: " << dstPath.PathString()); diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index f6d67ebee0e0..24ae8d41e8a1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -205,24 +205,32 @@ class TUploadSampleK: public TActorBootstrapped { } }; +TPath GetBuildPath(TSchemeShard* ss, const TIndexBuildInfo& buildInfo, const TString& tableName) { + return TPath::Init(buildInfo.TablePathId, ss) + .Dive(buildInfo.IndexName) + .Dive(tableName); +} + THolder LockPropose( - TSchemeShard* ss, const TIndexBuildInfo& buildInfo) + TSchemeShard* ss, const TIndexBuildInfo& buildInfo, TTxId txId, const TPath& path) { - auto propose = MakeHolder(ui64(buildInfo.LockTxId), ss->TabletID()); + auto propose = MakeHolder(ui64(txId), ss->TabletID()); propose->Record.SetFailOnExist(false); NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateLock); modifyScheme.SetInternal(true); - - TPath path = TPath::Init(buildInfo.TablePathId, ss); + modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); modifyScheme.SetWorkingDir(path.Parent().PathString()); modifyScheme.MutableLockConfig()->SetName(path.LeafName()); + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "LockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } -THolder InitiatePropose( +THolder CreateIndexPropose( TSchemeShard* ss, const TIndexBuildInfo& buildInfo) { auto propose = MakeHolder(ui64(buildInfo.InitiateTxId), ss->TabletID()); @@ -243,6 +251,9 @@ THolder InitiatePropose( Y_ABORT("Unknown operation kind while building InitiatePropose"); } + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "CreateIndexPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -259,10 +270,16 @@ THolder DropBuildPropose( NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetInternal(true); modifyScheme.SetWorkingDir(path.PathString()); + if (path.LockedBy()) { + modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); + } modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable); modifyScheme.MutableDrop()->SetName(buildInfo.KMeans.WriteTo(true)); + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "DropBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -294,10 +311,6 @@ THolder CreateBuildPropose( modifyScheme.ClearInitiateIndexBuild(); } - // TODO(mbkkt) for levels greater than zero we need to disable split/merge completely - // For now it's not guranteed, but very likely - // But lock is really unconvinient approach (needs to store TxId/etc) - // So maybe best way to do this is specify something in defintion, that will prevent these operations like IsBackup using namespace NTableIndex::NTableVectorKmeansTreeIndex; modifyScheme.SetWorkingDir(path.Dive(buildInfo.IndexName).PathString()); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpInitiateBuildIndexImplTable); @@ -322,6 +335,10 @@ THolder CreateBuildPropose( const auto shards = tableInfo->GetShard2PartitionIdx().size(); policy.SetMinPartitionsCount(shards); policy.SetMaxPartitionsCount(shards); + + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } op = CalcVectorKmeansTreePostingImplTableDesc({}, tableInfo, tableInfo->PartitionConfig(), implTableColumns, {}, suffix); @@ -342,6 +359,10 @@ THolder CreateBuildPropose( } policy.SetMinPartitionsCount(op.SplitBoundarySize() + 1); policy.SetMaxPartitionsCount(op.SplitBoundarySize() + 1); + + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -387,6 +408,9 @@ THolder AlterMainTablePropose( } + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "AlterMainTablePropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -399,9 +423,7 @@ THolder ApplyPropose( NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpApplyIndexBuild); modifyScheme.SetInternal(true); - modifyScheme.SetWorkingDir(TPath::Init(buildInfo.DomainPathId, ss).PathString()); - modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); auto& indexBuild = *modifyScheme.MutableApplyIndexBuild(); @@ -414,6 +436,9 @@ THolder ApplyPropose( indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId)); indexBuild.SetBuildIndexId(ui64(buildInfo.Id)); + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "ApplyPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -426,7 +451,6 @@ THolder UnlockPropose( NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropLock); modifyScheme.SetInternal(true); - modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); TPath path = TPath::Init(buildInfo.TablePathId, ss); @@ -435,6 +459,9 @@ THolder UnlockPropose( auto& lockConfig = *modifyScheme.MutableLockConfig(); lockConfig.SetName(path.LeafName()); + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "UnlockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -458,6 +485,9 @@ THolder CancelPropose( indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId)); indexBuild.SetBuildIndexId(ui64(buildInfo.Id)); + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "CancelPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -1045,7 +1075,9 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil } if (buildInfo.Shards.empty()) { NIceDb::TNiceDb db(txc.DB); - InitiateShards(db, buildInfo); + if (!InitiateShards(db, buildInfo)) { + return false; + } } switch (buildInfo.BuildKind) { case TIndexBuildInfo::EBuildKind::BuildSecondaryIndex: @@ -1080,7 +1112,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil case TIndexBuildInfo::EState::AlterMainTable: if (buildInfo.AlterMainTableTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.AlterMainTableTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), AlterMainTablePropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.AlterMainTableTxDone) { @@ -1092,9 +1124,9 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Locking: if (buildInfo.LockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.LockTxStatus == NKikimrScheme::StatusSuccess) { - Send(Self->SelfId(), LockPropose(Self, buildInfo), 0, ui64(BuildId)); + Send(Self->SelfId(), LockPropose(Self, buildInfo, buildInfo.LockTxId, TPath::Init(buildInfo.TablePathId, Self)), 0, ui64(BuildId)); } else if (!buildInfo.LockTxDone) { Send(Self->SelfId(), MakeHolder(ui64(buildInfo.LockTxId))); } else { @@ -1109,9 +1141,9 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Initiating: if (buildInfo.InitiateTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.InitiateTxStatus == NKikimrScheme::StatusSuccess) { - Send(Self->SelfId(), InitiatePropose(Self, buildInfo), 0, ui64(BuildId)); + Send(Self->SelfId(), CreateIndexPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.InitiateTxDone) { Send(Self->SelfId(), MakeHolder(ui64(buildInfo.InitiateTxId))); } else { @@ -1144,7 +1176,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Y_ASSERT(buildInfo.IsBuildVectorIndex()); Y_ASSERT(buildInfo.KMeans.Level > 2); if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), DropBuildPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1166,7 +1198,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil case TIndexBuildInfo::EState::CreateBuild: Y_ASSERT(buildInfo.IsBuildVectorIndex()); if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), CreateBuildPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1191,15 +1223,12 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil case TIndexBuildInfo::EState::LockBuild: Y_ASSERT(buildInfo.IsBuildVectorIndex()); if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { - Send(Self->SelfId(), CreateBuildPropose(Self, buildInfo), 0, ui64(BuildId)); + Send(Self->SelfId(), LockPropose(Self, buildInfo, buildInfo.ApplyTxId, GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom())), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { Send(Self->SelfId(), MakeHolder(ui64(buildInfo.ApplyTxId))); } else { - buildInfo.SnapshotTxId = {}; - buildInfo.SnapshotStep = {}; - buildInfo.ApplyTxId = {}; buildInfo.ApplyTxStatus = NKikimrScheme::StatusSuccess; buildInfo.ApplyTxDone = false; @@ -1215,7 +1244,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Applying: if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), ApplyPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1227,7 +1256,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Unlocking: if (buildInfo.UnlockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.UnlockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.UnlockTxDone) { @@ -1243,7 +1272,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Cancellation_Applying: if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), CancelPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1255,7 +1284,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Cancellation_Unlocking: if (buildInfo.UnlockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.UnlockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.UnlockTxDone) { @@ -1271,7 +1300,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Rejection_Applying: if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), CancelPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1283,7 +1312,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Rejection_Unlocking: if (buildInfo.UnlockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.UnlockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.UnlockTxDone) { @@ -1317,7 +1346,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil return TSerializedTableRange{{&from, 1}, false, {&to, 1}, true}; } - void InitiateShards(NIceDb::TNiceDb& db, TIndexBuildInfo& buildInfo) { + bool InitiateShards(NIceDb::TNiceDb& db, TIndexBuildInfo& buildInfo) { LOG_D("InitiateShards " << buildInfo.DebugString()); Y_ASSERT(buildInfo.Shards.empty()); @@ -1329,8 +1358,15 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil if (buildInfo.KMeans.Level == 1) { table = Self->Tables.at(buildInfo.TablePathId); } else { - auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); - table = Self->Tables.at(path.Dive(buildInfo.KMeans.ReadFrom())->PathId); + auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom()); + table = Self->Tables.at(path->PathId); + + if (!path.IsLocked()) { + // lock is needed to prevent its shards from beeing split + ChangeState(buildInfo.Id, TIndexBuildInfo::EState::LockBuild); + Progress(buildInfo.Id); + return false; + } } auto tableColumns = NTableIndex::ExtractInfo(table); // skip dropped columns TSerializedTableRange shardRange = InfiniteRange(tableColumns.Keys.size()); @@ -1351,6 +1387,8 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Self->PersistBuildIndexUploadInitiate(db, BuildId, x.ShardIdx, it->second); } + + return true; } void DoComplete(const TActorContext& ctx) override { diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp index ecda710d1ea7..9a3eecc50764 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp @@ -177,6 +177,11 @@ void TSchemeShard::TIndexBuilder::TTxBase::Send(TActorId dst, THolderTxAllocatorClient, MakeHolder(), 0, ui64(buildId)); +} + void TSchemeShard::TIndexBuilder::TTxBase::ChangeState(TIndexBuildId id, TIndexBuildInfo::EState state) { StateChanges.push_back(TChangeStateRec(id, state)); } diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h index 1a70dd31ec5e..02f96b34aef3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h @@ -32,6 +32,7 @@ class TSchemeShard::TIndexBuilder::TTxBase: public NTabletFlatExecutor::TTransac protected: void Send(TActorId dst, THolder message, ui32 flags = 0, ui64 cookie = 0); + void AllocateTxId(TIndexBuildId buildId); void ChangeState(TIndexBuildId id, TIndexBuildInfo::EState state); void Progress(TIndexBuildId id); void Fill(NKikimrIndexBuilder::TIndexBuild& index, const TIndexBuildInfo& indexInfo); diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index 01bae14754e7..c62f7be2d41f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -1875,6 +1875,10 @@ ui64 TPath::GetEffectiveACLVersion() const { return version; } +bool TPath::IsLocked() const { + return SS->LockedPaths.contains(Base()->PathId); +} + TTxId TPath::LockedBy() const { auto it = SS->LockedPaths.find(Base()->PathId); if (it != SS->LockedPaths.end()) { diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h index 4e167ac23dc3..071e488b7b86 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.h +++ b/ydb/core/tx/schemeshard/schemeshard_path.h @@ -183,6 +183,7 @@ class TPath { bool IsValidLeafName(TString& explain) const; TString GetEffectiveACL() const; ui64 GetEffectiveACLVersion() const; + bool IsLocked() const; TTxId LockedBy() const; bool IsActive() const; From 64b1450222fccf5b3183618883c311fbb241da20 Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 15 Apr 2025 18:27:38 +0300 Subject: [PATCH 05/14] fix lock --- .../schemeshard_build_index__progress.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index 24ae8d41e8a1..a9cea94152a0 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -265,17 +265,19 @@ THolder DropBuildPropose( auto propose = MakeHolder(ui64(buildInfo.ApplyTxId), ss->TabletID()); propose->Record.SetFailOnExist(true); - auto path = TPath::Init(buildInfo.TablePathId, ss).Dive(buildInfo.IndexName); + auto path = TPath::Init(buildInfo.TablePathId, ss) + .Dive(buildInfo.IndexName) + .Dive(buildInfo.KMeans.WriteTo(true)); NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetInternal(true); - modifyScheme.SetWorkingDir(path.PathString()); - if (path.LockedBy()) { + modifyScheme.SetWorkingDir(path.Parent().PathString()); + if (path.IsLocked()) { modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); } modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable); - modifyScheme.MutableDrop()->SetName(buildInfo.KMeans.WriteTo(true)); + modifyScheme.MutableDrop()->SetName(path->Name); LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "DropBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); @@ -474,9 +476,7 @@ THolder CancelPropose( NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCancelIndexBuild); modifyScheme.SetInternal(true); - modifyScheme.SetWorkingDir(TPath::Init(buildInfo.DomainPathId, ss).PathString()); - modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); auto& indexBuild = *modifyScheme.MutableCancelIndexBuild(); @@ -1361,12 +1361,12 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom()); table = Self->Tables.at(path->PathId); - if (!path.IsLocked()) { - // lock is needed to prevent its shards from beeing split + if (!path.IsLocked()) { // lock is needed to prevent its shards from beeing split ChangeState(buildInfo.Id, TIndexBuildInfo::EState::LockBuild); Progress(buildInfo.Id); return false; } + Y_ASSERT(path.LockedBy() == buildInfo.LockTxId); } auto tableColumns = NTableIndex::ExtractInfo(table); // skip dropped columns TSerializedTableRange shardRange = InfiniteRange(tableColumns.Keys.size()); From c1198266544c47f228094e91950bf51544491ccf Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 15 Apr 2025 18:45:19 +0300 Subject: [PATCH 06/14] use GetBuildPath --- .../schemeshard_build_index__progress.cpp | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index a9cea94152a0..ca682ac5103f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -265,9 +265,7 @@ THolder DropBuildPropose( auto propose = MakeHolder(ui64(buildInfo.ApplyTxId), ss->TabletID()); propose->Record.SetFailOnExist(true); - auto path = TPath::Init(buildInfo.TablePathId, ss) - .Dive(buildInfo.IndexName) - .Dive(buildInfo.KMeans.WriteTo(true)); + auto path = GetBuildPath(ss, buildInfo, buildInfo.KMeans.WriteTo(true)); NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetInternal(true); @@ -534,8 +532,8 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil if (buildInfo.KMeans.Level == 1) { buildInfo.TablePathId.ToProto(ev->Record.MutablePathId()); } else { - auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); - path.Dive(buildInfo.KMeans.ReadFrom())->PathId.ToProto(ev->Record.MutablePathId()); + auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom()); + path->PathId.ToProto(ev->Record.MutablePathId()); } ev->Record.SetK(buildInfo.KMeans.K); @@ -657,8 +655,8 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil auto ev = MakeHolder(); ev->Record.SetId(ui64(BuildId)); - auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); - path.Dive(buildInfo.KMeans.ReadFrom())->PathId.ToProto(ev->Record.MutablePathId()); + auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom()); + path->PathId.ToProto(ev->Record.MutablePathId()); path.Rise(); *ev->Record.MutableSettings() = std::get( buildInfo.SpecializedIndexDescription).GetSettings().settings(); @@ -701,7 +699,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil buildInfo.SerializeToProto(Self, ev->Record.MutableColumnBuildSettings()); } else { if (buildInfo.TargetName.empty()) { - TPath implTable = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName).Dive( + TPath implTable = GetBuildPath(Self, buildInfo, buildInfo.IsBuildPrefixedVectorIndex() ? buildInfo.KMeans.WriteTo() : NTableIndex::ImplTable); buildInfo.TargetName = implTable.PathString(); @@ -743,9 +741,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil void SendUploadSampleKRequest(TIndexBuildInfo& buildInfo) { buildInfo.Sample.MakeStrictTop(buildInfo.KMeans.K); - auto path = TPath::Init(buildInfo.TablePathId, Self) - .Dive(buildInfo.IndexName) - .Dive(NTableIndex::NTableVectorKmeansTreeIndex::LevelTable); + auto path = GetBuildPath(Self, buildInfo, NTableIndex::NTableVectorKmeansTreeIndex::LevelTable); Y_ASSERT(buildInfo.Sample.Rows.size() <= buildInfo.KMeans.K); auto actor = new TUploadSampleK(path.PathString(), buildInfo.ScanSettings, Self->SelfId(), ui64(BuildId), @@ -1361,7 +1357,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom()); table = Self->Tables.at(path->PathId); - if (!path.IsLocked()) { // lock is needed to prevent its shards from beeing split + if (!path.IsLocked()) { // lock is needed to prevent table shards from beeing split ChangeState(buildInfo.Id, TIndexBuildInfo::EState::LockBuild); Progress(buildInfo.Id); return false; From 86b355d14691d3eec8ee09e563f0de897a5b969b Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 16 Apr 2025 08:11:38 +0300 Subject: [PATCH 07/14] skip locked tables faster --- ydb/core/tx/schemeshard/schemeshard__table_stats.cpp | 6 ++++++ .../tx/schemeshard/schemeshard__table_stats_histogram.cpp | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp index 9dfaac175603..4955b193aa89 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp @@ -509,6 +509,12 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId, return true; } + if (auto lock = Self->LockedPaths.FindPtr(pathId); lock) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Postpone split tablet " << datashardId << " because it is locked by " << *lock); + return true; + } + // Request histograms from the datashard LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Requesting full tablet stats " << datashardId << " to split it"); diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp index f78ff9771654..dbab56bb3a75 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp @@ -355,6 +355,11 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex return true; } + if (auto lock = Self->LockedPaths.FindPtr(tableId); lock) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxPartitionHistogram Skip locked table tablet " << datashardId << " by " << *lock); + } + auto shardIdx = Self->TabletIdToShardIdx[datashardId]; const auto forceShardSplitSettings = Self->SplitSettings.GetForceShardSplitSettings(); From f6cf0b5d9c45725b5a1f8e97d8c9a19d5ca2180e Mon Sep 17 00:00:00 2001 From: kungasc Date: Thu, 17 Apr 2025 13:18:12 +0300 Subject: [PATCH 08/14] fix missing return --- ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp index dbab56bb3a75..563f2c9d05b1 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp @@ -358,6 +358,7 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex if (auto lock = Self->LockedPaths.FindPtr(tableId); lock) { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxPartitionHistogram Skip locked table tablet " << datashardId << " by " << *lock); + return true; } auto shardIdx = Self->TabletIdToShardIdx[datashardId]; From abb35b5e1003012359a8f704df3447823f9bbf7f Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 22 Apr 2025 10:57:12 +0300 Subject: [PATCH 09/14] fix typo --- ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index ca682ac5103f..5406e109fd74 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -1357,7 +1357,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom()); table = Self->Tables.at(path->PathId); - if (!path.IsLocked()) { // lock is needed to prevent table shards from beeing split + if (!path.IsLocked()) { // lock is needed to prevent table shards from being split ChangeState(buildInfo.Id, TIndexBuildInfo::EState::LockBuild); Progress(buildInfo.Id); return false; From 59b5938a2168e93390ac110e325cc64480cc7c23 Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 22 Apr 2025 10:59:49 +0300 Subject: [PATCH 10/14] cr fix error --- ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index 5406e109fd74..a4e300cc1f7a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -248,7 +248,7 @@ THolder CreateIndexPropose( modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateColumnBuild); buildInfo.SerializeToProto(ss, modifyScheme.MutableInitiateColumnBuild()); } else { - Y_ABORT("Unknown operation kind while building InitiatePropose"); + Y_ABORT("Unknown operation kind while building CreateIndexPropose"); } LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, From 85df583d5b21f5047dc438c27d11baa61e580903 Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 22 Apr 2025 11:08:24 +0300 Subject: [PATCH 11/14] cr use IsLocked method --- .../schemeshard/schemeshard__table_stats.cpp | 32 ++++++++----------- .../schemeshard__table_stats_histogram.cpp | 5 +-- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp index 4955b193aa89..61f3bb7a0825 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp @@ -484,22 +484,18 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId, return true; } - { - auto path = TPath::Init(pathId, Self); - auto checks = path.Check(); - - constexpr ui64 deltaShards = 2; - checks - .PathShardsLimit(deltaShards) - .ShardsLimit(deltaShards); - - if (!checks) { - LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Do not request full stats from datashard" - << ", datashard: " << datashardId - << ", reason: " << checks.GetError()); - return true; - } + auto path = TPath::Init(pathId, Self); + auto checks = path.Check(); + constexpr ui64 deltaShards = 2; + checks + .PathShardsLimit(deltaShards) + .ShardsLimit(deltaShards); + if (!checks) { + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Do not request full stats from datashard" + << ", datashard: " << datashardId + << ", reason: " << checks.GetError()); + return true; } if (newStats.HasBorrowedData) { @@ -509,9 +505,9 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId, return true; } - if (auto lock = Self->LockedPaths.FindPtr(pathId); lock) { + if (path.IsLocked()) { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Postpone split tablet " << datashardId << " because it is locked by " << *lock); + "Postpone split tablet " << datashardId << " because it is locked by " << path.LockedBy()); return true; } diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp index 563f2c9d05b1..7ec9bc887f48 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp @@ -341,6 +341,7 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex } TTableInfo::TPtr table = Self->Tables[tableId]; + auto path = TPath::Init(tableId, Self); if (!Self->TabletIdToShardIdx.contains(datashardId)) { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -355,9 +356,9 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex return true; } - if (auto lock = Self->LockedPaths.FindPtr(tableId); lock) { + if (path.IsLocked()) { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "TTxPartitionHistogram Skip locked table tablet " << datashardId << " by " << *lock); + "TTxPartitionHistogram Skip locked table tablet " << datashardId << " by " << path.LockedBy()); return true; } From 1b50895439b8cc164605a66eadbd1e5ae456eea3 Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 23 Apr 2025 17:08:36 +0300 Subject: [PATCH 12/14] cr better comment --- .../schemeshard/schemeshard__operation_apply_build_index.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp index 08923fa797bc..ce8f4a7c5433 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp @@ -48,7 +48,9 @@ ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& n } rejected = false; auto transaction = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable); - if (implTable.IsLocked()) { // some impl tables may be not locked + if (implTable.IsLocked()) { + // because some impl tables may be not locked, do not pass lock guard for them + // otherwise `CheckLocks` check would fail *transaction.MutableLockGuard() = lockGuard; } auto operation = transaction.MutableDrop(); From 997787b2b764a3ab1b2c9fc2966614221ae59b02 Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 23 Apr 2025 17:39:29 +0300 Subject: [PATCH 13/14] cr pass lock in lock config --- ydb/core/protos/flat_scheme_op.proto | 1 + .../tx/schemeshard/schemeshard__operation_create_lock.cpp | 4 ++-- ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index c6b048d3d9d0..5549e2a85d05 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1586,6 +1586,7 @@ message TIndexBuildControl { message TLockConfig { optional string Name = 1; + optional uint64 LockTxId = 2; // if missing, current tx id is used } message TLockGuard { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp index d78d4271c57a..f3234af41680 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp @@ -95,8 +95,8 @@ class TCreateLock: public TSubOperation { THolder Propose(const TString&, TOperationContext& context) override { const auto& workingDir = Transaction.GetWorkingDir(); const auto& op = Transaction.GetLockConfig(); - const auto lockTxId = Transaction.HasLockGuard() && Transaction.GetLockGuard().HasOwnerTxId() - ? TTxId(Transaction.GetLockGuard().GetOwnerTxId()) + const TTxId lockTxId = op.HasLockTxId() + ? TTxId(op.GetLockTxId()) : OperationId.GetTxId(); LOG_N("TCreateLock Propose" diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index a4e300cc1f7a..f06974855a0d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -220,9 +220,9 @@ THolder LockPropose( NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateLock); modifyScheme.SetInternal(true); - modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); modifyScheme.SetWorkingDir(path.Parent().PathString()); modifyScheme.MutableLockConfig()->SetName(path.LeafName()); + modifyScheme.MutableLockConfig()->SetLockTxId(ui64(buildInfo.LockTxId)); LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "LockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); @@ -271,6 +271,8 @@ THolder DropBuildPropose( modifyScheme.SetInternal(true); modifyScheme.SetWorkingDir(path.Parent().PathString()); if (path.IsLocked()) { + // because some impl tables may be not locked, do not pass lock guard for them + // otherwise `CheckLocks` check would fail modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); } From 975d43575923c03532655b8582a4c1028dfd3ddd Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 23 Apr 2025 17:49:16 +0300 Subject: [PATCH 14/14] cr return old check locks call --- .../schemeshard__operation_create_lock.cpp | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp index f3234af41680..07efe6838fa7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp @@ -161,19 +161,16 @@ class TCreateLock: public TSubOperation { const auto pathId = tablePath.Base()->PathId; result->SetPathId(pathId.LocalPathId); - if (auto lockedBy = tablePath.LockedBy(); lockedBy != InvalidTxId) { - if (lockedBy == lockTxId) { - result->SetError(NKikimrScheme::StatusAlreadyExists, TStringBuilder() << "path checks failed" - << ", path already locked by this operation" - << ", path: " << tablePath.PathString()); - return result; - } else { - result->SetError(NKikimrScheme::StatusMultipleModifications, TStringBuilder() << "path checks failed" - << ", path already locked by another operation" - << ", path: " << tablePath.PathString() - << ", locked by: " << lockedBy); - return result; - } + if (tablePath.LockedBy() == lockTxId) { + result->SetError(NKikimrScheme::StatusAlreadyExists, TStringBuilder() << "path checks failed" + << ", path already locked by this operation" + << ", path: " << tablePath.PathString()); + return result; + } + TString errStr; + if (!context.SS->CheckLocks(pathId, Transaction, errStr)) { + result->SetError(NKikimrScheme::StatusMultipleModifications, errStr); + return result; } auto guard = context.DbGuard();