diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp index 248c9ebb9b4b..bcddfee4385d 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp @@ -119,6 +119,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto& server = kikimr.GetTestServer(); auto gateway = GetIcGateway(server); @@ -275,6 +277,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto& server = kikimr.GetTestServer(); auto gateway = GetIcGateway(server); @@ -389,6 +393,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -447,6 +453,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -527,6 +535,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -607,6 +617,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -792,6 +804,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -915,6 +929,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1047,6 +1063,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1282,6 +1300,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({ setting }); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1377,6 +1397,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1514,6 +1536,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1707,6 +1731,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1797,6 +1823,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1863,6 +1891,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); TScriptingClient client(kikimr.GetDriver()); { @@ -1904,6 +1934,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -2382,6 +2414,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, false); { @@ -2425,6 +2459,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, false); { @@ -2468,6 +2504,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, true); { @@ -2511,6 +2549,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, true); { @@ -2554,6 +2594,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, false); { @@ -2597,6 +2639,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, false); { @@ -2640,6 +2684,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, true); { @@ -2683,6 +2729,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForVectorIndex(db, true); { @@ -2716,6 +2764,52 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { DoPositiveQueriesVectorIndexOrderByCosine(session); } + Y_UNIT_TEST(SimpleVectorIndexOrderByCosineDistanceNotNullableLevel3) { + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableVectorIndex(true); + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetFeatureFlags(featureFlags) + .SetKqpSettings({setting}); + + TKikimrRunner kikimr(serverSettings); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + + auto db = kikimr.GetTableClient(); + auto session = DoCreateTableForVectorIndex(db, false); + { + const TString createIndex(Q_(R"( + ALTER TABLE `/Root/TestTable` + ADD INDEX index + GLOBAL USING vector_kmeans_tree + ON (emb) + WITH (distance=cosine, vector_type="uint8", vector_dimension=2, levels=3, clusters=2); + )")); + + auto result = session.ExecuteSchemeQuery(createIndex) + .ExtractValueSync(); + + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + { + auto result = session.DescribeTable("/Root/TestTable").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); + const auto& indexes = result.GetTableDescription().GetIndexDescriptions(); + UNIT_ASSERT_EQUAL(indexes.size(), 1); + UNIT_ASSERT_EQUAL(indexes[0].GetIndexName(), "index"); + UNIT_ASSERT_EQUAL(indexes[0].GetIndexColumns(), std::vector{"emb"}); + const auto& settings = std::get(indexes[0].GetIndexSettings()); + UNIT_ASSERT_EQUAL(settings.Settings.Metric, NYdb::NTable::TVectorIndexSettings::EMetric::CosineDistance); + UNIT_ASSERT_EQUAL(settings.Settings.VectorType, NYdb::NTable::TVectorIndexSettings::EVectorType::Uint8); + UNIT_ASSERT_EQUAL(settings.Settings.VectorDimension, 2); + UNIT_ASSERT_EQUAL(settings.Levels, 3); + UNIT_ASSERT_EQUAL(settings.Clusters, 2); + } + // TODO: fix somehow? + // DoPositiveQueriesVectorIndexOrderByCosine(session); + } + void DoPositiveQueriesPrefixedVectorIndexOrderBy( TSession& session, std::string_view function, @@ -2888,6 +2982,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, false); @@ -2933,6 +3029,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, false); @@ -2978,6 +3076,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, true); @@ -3023,6 +3123,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, true); @@ -3068,6 +3170,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, false); @@ -3205,6 +3309,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, false); @@ -3250,6 +3356,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, true); @@ -3295,6 +3403,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = DoCreateTableForPrefixedVectorIndex(db, true); @@ -3408,6 +3518,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3485,6 +3597,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3636,6 +3750,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { .SetAppConfig(app); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3714,6 +3830,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3762,6 +3880,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -3826,6 +3946,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -4017,6 +4139,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -4144,6 +4268,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(app); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -4484,6 +4610,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(app); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -4760,6 +4888,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -4998,6 +5128,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5160,6 +5292,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5247,6 +5381,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto scheme = kikimr.GetSchemeClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5297,6 +5433,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5337,6 +5475,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5424,6 +5564,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5622,6 +5764,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda serverSettings.SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTablesWithIndex(session); @@ -5698,6 +5842,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTablesWithIndex(session); @@ -5775,6 +5921,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -5965,6 +6113,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + TScriptingClient client(kikimr.GetDriver()); auto scriptResult = client.ExecuteYqlScript(R"( @@ -6058,6 +6208,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -6115,6 +6267,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -6166,6 +6320,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetKqpSettings({setting}); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -6712,6 +6868,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda .SetAppConfig(appConfig); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index c6b048d3d9d0..5549e2a85d05 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1586,6 +1586,7 @@ message TIndexBuildControl { message TLockConfig { optional string Name = 1; + optional uint64 LockTxId = 2; // if missing, current tx id is used } message TLockGuard { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp index 06a5c1fa3e66..ce8f4a7c5433 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp @@ -30,7 +30,7 @@ ISubOperation::TPtr FinalizeIndexImplTable(TOperationContext& context, const TPa return CreateFinalizeBuildIndexImplTable(partId, transaction); } -ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& nextId, const TOperationId& partId, const TString& name, const TPathId& pathId, bool& rejected) { +ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& nextId, const TOperationId& partId, const TString& name, const TPathId& pathId, const NKikimrSchemeOp::TLockGuard& lockGuard, bool& rejected) { TPath implTable = index.Child(name); Y_ABORT_UNLESS(implTable->PathId == pathId); Y_ABORT_UNLESS(implTable.LeafName() == name); @@ -48,6 +48,11 @@ ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& n } rejected = false; auto transaction = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable); + if (implTable.IsLocked()) { + // because some impl tables may be not locked, do not pass lock guard for them + // otherwise `CheckLocks` check would fail + *transaction.MutableLockGuard() = lockGuard; + } auto operation = transaction.MutableDrop(); operation->SetName(name); return CreateDropTable(partId, transaction); @@ -98,7 +103,7 @@ TVector ApplyBuildIndex(TOperationId nextId, const TTxTrans const auto partId = NextPartId(nextId, result); if (NTableIndex::IsBuildImplTable(indexImplTableName)) { bool rejected = false; - auto op = DropIndexImplTable(index, nextId, partId, indexImplTableName, indexChildItems.second, rejected); + auto op = DropIndexImplTable(index, nextId, partId, indexImplTableName, indexChildItems.second, tx.GetLockGuard(), rejected); if (rejected) { return {std::move(op)}; } @@ -153,7 +158,7 @@ TVector CancelBuildIndex(TOperationId nextId, const TTxTran for (auto& indexChildItems : index.Base()->GetChildren()) { const auto partId = NextPartId(nextId, result); bool rejected = false; - auto op = DropIndexImplTable(index, nextId, partId, indexChildItems.first, indexChildItems.second, rejected); + auto op = DropIndexImplTable(index, nextId, partId, indexChildItems.first, indexChildItems.second, tx.GetLockGuard(), rejected); if (rejected) { return {std::move(op)}; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp index e8c02b30c81d..07efe6838fa7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp @@ -95,6 +95,9 @@ class TCreateLock: public TSubOperation { THolder Propose(const TString&, TOperationContext& context) override { const auto& workingDir = Transaction.GetWorkingDir(); const auto& op = Transaction.GetLockConfig(); + const TTxId lockTxId = op.HasLockTxId() + ? TTxId(op.GetLockTxId()) + : OperationId.GetTxId(); LOG_N("TCreateLock Propose" << ": opId# " << OperationId @@ -158,13 +161,12 @@ class TCreateLock: public TSubOperation { const auto pathId = tablePath.Base()->PathId; result->SetPathId(pathId.LocalPathId); - if (tablePath.LockedBy() == OperationId.GetTxId()) { + if (tablePath.LockedBy() == lockTxId) { result->SetError(NKikimrScheme::StatusAlreadyExists, TStringBuilder() << "path checks failed" << ", path already locked by this operation" << ", path: " << tablePath.PathString()); return result; } - TString errStr; if (!context.SS->CheckLocks(pathId, Transaction, errStr)) { result->SetError(NKikimrScheme::StatusMultipleModifications, errStr); @@ -177,7 +179,7 @@ class TCreateLock: public TSubOperation { context.MemChanges.GrabNewTxState(context.SS, OperationId); context.DbChanges.PersistPath(pathId); - context.DbChanges.PersistLongLock(pathId, OperationId.GetTxId()); + context.DbChanges.PersistLongLock(pathId, lockTxId); context.DbChanges.PersistTxState(OperationId); Y_ABORT_UNLESS(!context.SS->FindTx(OperationId)); @@ -194,7 +196,7 @@ class TCreateLock: public TSubOperation { context.OnComplete.Dependence(splitOpId.GetTxId(), OperationId.GetTxId()); } - context.SS->LockedPaths[pathId] = OperationId.GetTxId(); + context.SS->LockedPaths[pathId] = lockTxId; context.SS->TabletCounters->Simple()[COUNTER_LOCKS_COUNT].Add(1); context.OnComplete.ActivateTx(OperationId); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp index 8fdfeb0a7d09..4e3cc7c79314 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp @@ -164,7 +164,7 @@ class TDropLock: public TSubOperation { const auto pathId = dstPath.Base()->PathId; result->SetPathId(pathId.LocalPathId); - if (!dstPath.LockedBy()) { + if (!dstPath.IsLocked()) { result->SetError(TEvSchemeShard::EStatus::StatusAlreadyExists, TStringBuilder() << "path checks failed" << ", path already unlocked" << ", path: " << dstPath.PathString()); diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp index 9dfaac175603..61f3bb7a0825 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp @@ -484,22 +484,18 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId, return true; } - { - auto path = TPath::Init(pathId, Self); - auto checks = path.Check(); - - constexpr ui64 deltaShards = 2; - checks - .PathShardsLimit(deltaShards) - .ShardsLimit(deltaShards); - - if (!checks) { - LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "Do not request full stats from datashard" - << ", datashard: " << datashardId - << ", reason: " << checks.GetError()); - return true; - } + auto path = TPath::Init(pathId, Self); + auto checks = path.Check(); + constexpr ui64 deltaShards = 2; + checks + .PathShardsLimit(deltaShards) + .ShardsLimit(deltaShards); + if (!checks) { + LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Do not request full stats from datashard" + << ", datashard: " << datashardId + << ", reason: " << checks.GetError()); + return true; } if (newStats.HasBorrowedData) { @@ -509,6 +505,12 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId, return true; } + if (path.IsLocked()) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Postpone split tablet " << datashardId << " because it is locked by " << path.LockedBy()); + return true; + } + // Request histograms from the datashard LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Requesting full tablet stats " << datashardId << " to split it"); diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp index f78ff9771654..7ec9bc887f48 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp @@ -341,6 +341,7 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex } TTableInfo::TPtr table = Self->Tables[tableId]; + auto path = TPath::Init(tableId, Self); if (!Self->TabletIdToShardIdx.contains(datashardId)) { LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -355,6 +356,12 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex return true; } + if (path.IsLocked()) { + LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TTxPartitionHistogram Skip locked table tablet " << datashardId << " by " << path.LockedBy()); + return true; + } + auto shardIdx = Self->TabletIdToShardIdx[datashardId]; const auto forceShardSplitSettings = Self->SplitSettings.GetForceShardSplitSettings(); diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index 7e2df90002e8..f06974855a0d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -22,45 +22,6 @@ namespace NKikimr { namespace NSchemeShard { -static constexpr const char* Name(TIndexBuildInfo::EState state) noexcept { - switch (state) { - case TIndexBuildInfo::EState::Invalid: - return "Invalid"; - case TIndexBuildInfo::EState::AlterMainTable: - return "AlterMainTable"; - case TIndexBuildInfo::EState::Locking: - return "Locking"; - case TIndexBuildInfo::EState::GatheringStatistics: - return "GatheringStatistics"; - case TIndexBuildInfo::EState::Initiating: - return "Initiating"; - case TIndexBuildInfo::EState::Filling: - return "Filling"; - case TIndexBuildInfo::EState::DropBuild: - return "DropBuild"; - case TIndexBuildInfo::EState::CreateBuild: - return "CreateBuild"; - case TIndexBuildInfo::EState::Applying: - return "Applying"; - case TIndexBuildInfo::EState::Unlocking: - return "Unlocking"; - case TIndexBuildInfo::EState::Done: - return "Done"; - case TIndexBuildInfo::EState::Cancellation_Applying: - return "Cancellation_Applying"; - case TIndexBuildInfo::EState::Cancellation_Unlocking: - return "Cancellation_Unlocking"; - case TIndexBuildInfo::EState::Cancelled: - return "Cancelled"; - case TIndexBuildInfo::EState::Rejection_Applying: - return "Rejection_Applying"; - case TIndexBuildInfo::EState::Rejection_Unlocking: - return "Rejection_Unlocking"; - case TIndexBuildInfo::EState::Rejected: - return "Rejected"; - } -} - // return count, parts, step static std::tuple ComputeKMeansBoundaries(const NSchemeShard::TTableInfo& tableInfo, const TIndexBuildInfo& buildInfo) { const auto& kmeans = buildInfo.KMeans; @@ -244,24 +205,32 @@ class TUploadSampleK: public TActorBootstrapped { } }; +TPath GetBuildPath(TSchemeShard* ss, const TIndexBuildInfo& buildInfo, const TString& tableName) { + return TPath::Init(buildInfo.TablePathId, ss) + .Dive(buildInfo.IndexName) + .Dive(tableName); +} + THolder LockPropose( - TSchemeShard* ss, const TIndexBuildInfo& buildInfo) + TSchemeShard* ss, const TIndexBuildInfo& buildInfo, TTxId txId, const TPath& path) { - auto propose = MakeHolder(ui64(buildInfo.LockTxId), ss->TabletID()); + auto propose = MakeHolder(ui64(txId), ss->TabletID()); propose->Record.SetFailOnExist(false); NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateLock); modifyScheme.SetInternal(true); - - TPath path = TPath::Init(buildInfo.TablePathId, ss); modifyScheme.SetWorkingDir(path.Parent().PathString()); modifyScheme.MutableLockConfig()->SetName(path.LeafName()); + modifyScheme.MutableLockConfig()->SetLockTxId(ui64(buildInfo.LockTxId)); + + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "LockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; } -THolder InitiatePropose( +THolder CreateIndexPropose( TSchemeShard* ss, const TIndexBuildInfo& buildInfo) { auto propose = MakeHolder(ui64(buildInfo.InitiateTxId), ss->TabletID()); @@ -279,9 +248,12 @@ THolder InitiatePropose( modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateColumnBuild); buildInfo.SerializeToProto(ss, modifyScheme.MutableInitiateColumnBuild()); } else { - Y_ABORT("Unknown operation kind while building InitiatePropose"); + Y_ABORT("Unknown operation kind while building CreateIndexPropose"); } + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "CreateIndexPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -293,14 +265,22 @@ THolder DropBuildPropose( auto propose = MakeHolder(ui64(buildInfo.ApplyTxId), ss->TabletID()); propose->Record.SetFailOnExist(true); - auto path = TPath::Init(buildInfo.TablePathId, ss).Dive(buildInfo.IndexName); + auto path = GetBuildPath(ss, buildInfo, buildInfo.KMeans.WriteTo(true)); NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetInternal(true); - modifyScheme.SetWorkingDir(path.PathString()); + modifyScheme.SetWorkingDir(path.Parent().PathString()); + if (path.IsLocked()) { + // because some impl tables may be not locked, do not pass lock guard for them + // otherwise `CheckLocks` check would fail + modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); + } modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable); - modifyScheme.MutableDrop()->SetName(buildInfo.KMeans.WriteTo(true)); + modifyScheme.MutableDrop()->SetName(path->Name); + + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "DropBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; } @@ -333,10 +313,6 @@ THolder CreateBuildPropose( modifyScheme.ClearInitiateIndexBuild(); } - // TODO(mbkkt) for levels greater than zero we need to disable split/merge completely - // For now it's not guranteed, but very likely - // But lock is really unconvinient approach (needs to store TxId/etc) - // So maybe best way to do this is specify something in defintion, that will prevent these operations like IsBackup using namespace NTableIndex::NTableVectorKmeansTreeIndex; modifyScheme.SetWorkingDir(path.Dive(buildInfo.IndexName).PathString()); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpInitiateBuildIndexImplTable); @@ -361,6 +337,10 @@ THolder CreateBuildPropose( const auto shards = tableInfo->GetShard2PartitionIdx().size(); policy.SetMinPartitionsCount(shards); policy.SetMaxPartitionsCount(shards); + + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } op = CalcVectorKmeansTreePostingImplTableDesc({}, tableInfo, tableInfo->PartitionConfig(), implTableColumns, {}, suffix); @@ -381,6 +361,10 @@ THolder CreateBuildPropose( } policy.SetMinPartitionsCount(op.SplitBoundarySize() + 1); policy.SetMaxPartitionsCount(op.SplitBoundarySize() + 1); + + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -426,6 +410,9 @@ THolder AlterMainTablePropose( } + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "AlterMainTablePropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -438,9 +425,7 @@ THolder ApplyPropose( NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpApplyIndexBuild); modifyScheme.SetInternal(true); - modifyScheme.SetWorkingDir(TPath::Init(buildInfo.DomainPathId, ss).PathString()); - modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); auto& indexBuild = *modifyScheme.MutableApplyIndexBuild(); @@ -453,6 +438,9 @@ THolder ApplyPropose( indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId)); indexBuild.SetBuildIndexId(ui64(buildInfo.Id)); + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "ApplyPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -465,7 +453,6 @@ THolder UnlockPropose( NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropLock); modifyScheme.SetInternal(true); - modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); TPath path = TPath::Init(buildInfo.TablePathId, ss); @@ -474,6 +461,9 @@ THolder UnlockPropose( auto& lockConfig = *modifyScheme.MutableLockConfig(); lockConfig.SetName(path.LeafName()); + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "UnlockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -486,9 +476,7 @@ THolder CancelPropose( NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCancelIndexBuild); modifyScheme.SetInternal(true); - modifyScheme.SetWorkingDir(TPath::Init(buildInfo.DomainPathId, ss).PathString()); - modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId)); auto& indexBuild = *modifyScheme.MutableCancelIndexBuild(); @@ -497,6 +485,9 @@ THolder CancelPropose( indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId)); indexBuild.SetBuildIndexId(ui64(buildInfo.Id)); + LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "CancelPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + return propose; } @@ -543,8 +534,8 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil if (buildInfo.KMeans.Level == 1) { buildInfo.TablePathId.ToProto(ev->Record.MutablePathId()); } else { - auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); - path.Dive(buildInfo.KMeans.ReadFrom())->PathId.ToProto(ev->Record.MutablePathId()); + auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom()); + path->PathId.ToProto(ev->Record.MutablePathId()); } ev->Record.SetK(buildInfo.KMeans.K); @@ -666,8 +657,8 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil auto ev = MakeHolder(); ev->Record.SetId(ui64(BuildId)); - auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); - path.Dive(buildInfo.KMeans.ReadFrom())->PathId.ToProto(ev->Record.MutablePathId()); + auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom()); + path->PathId.ToProto(ev->Record.MutablePathId()); path.Rise(); *ev->Record.MutableSettings() = std::get( buildInfo.SpecializedIndexDescription).GetSettings().settings(); @@ -710,7 +701,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil buildInfo.SerializeToProto(Self, ev->Record.MutableColumnBuildSettings()); } else { if (buildInfo.TargetName.empty()) { - TPath implTable = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName).Dive( + TPath implTable = GetBuildPath(Self, buildInfo, buildInfo.IsBuildPrefixedVectorIndex() ? buildInfo.KMeans.WriteTo() : NTableIndex::ImplTable); buildInfo.TargetName = implTable.PathString(); @@ -752,9 +743,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil void SendUploadSampleKRequest(TIndexBuildInfo& buildInfo) { buildInfo.Sample.MakeStrictTop(buildInfo.KMeans.K); - auto path = TPath::Init(buildInfo.TablePathId, Self) - .Dive(buildInfo.IndexName) - .Dive(NTableIndex::NTableVectorKmeansTreeIndex::LevelTable); + auto path = GetBuildPath(Self, buildInfo, NTableIndex::NTableVectorKmeansTreeIndex::LevelTable); Y_ASSERT(buildInfo.Sample.Rows.size() <= buildInfo.KMeans.K); auto actor = new TUploadSampleK(path.PathString(), buildInfo.ScanSettings, Self->SelfId(), ui64(BuildId), @@ -1084,7 +1073,9 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil } if (buildInfo.Shards.empty()) { NIceDb::TNiceDb db(txc.DB); - InitiateShards(db, buildInfo); + if (!InitiateShards(db, buildInfo)) { + return false; + } } switch (buildInfo.BuildKind) { case TIndexBuildInfo::EBuildKind::BuildSecondaryIndex: @@ -1111,8 +1102,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Y_ABORT_UNLESS(buildInfoPtr); auto& buildInfo = *buildInfoPtr->Get(); - LOG_I("TTxBuildProgress: Resume: id# " << BuildId); - LOG_D("TTxBuildProgress: Resume: " << buildInfo); + LOG_I("TTxBuildProgress: Execute: " << BuildId << " " << buildInfo.State << " " << buildInfo); switch (buildInfo.State) { case TIndexBuildInfo::EState::Invalid: @@ -1120,7 +1110,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil case TIndexBuildInfo::EState::AlterMainTable: if (buildInfo.AlterMainTableTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.AlterMainTableTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), AlterMainTablePropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.AlterMainTableTxDone) { @@ -1132,9 +1122,9 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Locking: if (buildInfo.LockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.LockTxStatus == NKikimrScheme::StatusSuccess) { - Send(Self->SelfId(), LockPropose(Self, buildInfo), 0, ui64(BuildId)); + Send(Self->SelfId(), LockPropose(Self, buildInfo, buildInfo.LockTxId, TPath::Init(buildInfo.TablePathId, Self)), 0, ui64(BuildId)); } else if (!buildInfo.LockTxDone) { Send(Self->SelfId(), MakeHolder(ui64(buildInfo.LockTxId))); } else { @@ -1149,9 +1139,9 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Initiating: if (buildInfo.InitiateTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.InitiateTxStatus == NKikimrScheme::StatusSuccess) { - Send(Self->SelfId(), InitiatePropose(Self, buildInfo), 0, ui64(BuildId)); + Send(Self->SelfId(), CreateIndexPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.InitiateTxDone) { Send(Self->SelfId(), MakeHolder(ui64(buildInfo.InitiateTxId))); } else { @@ -1184,7 +1174,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Y_ASSERT(buildInfo.IsBuildVectorIndex()); Y_ASSERT(buildInfo.KMeans.Level > 2); if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), DropBuildPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1206,7 +1196,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil case TIndexBuildInfo::EState::CreateBuild: Y_ASSERT(buildInfo.IsBuildVectorIndex()); if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), CreateBuildPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1228,9 +1218,31 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Progress(BuildId); } break; + case TIndexBuildInfo::EState::LockBuild: + Y_ASSERT(buildInfo.IsBuildVectorIndex()); + if (buildInfo.ApplyTxId == InvalidTxId) { + AllocateTxId(BuildId); + } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { + Send(Self->SelfId(), LockPropose(Self, buildInfo, buildInfo.ApplyTxId, GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom())), 0, ui64(BuildId)); + } else if (!buildInfo.ApplyTxDone) { + Send(Self->SelfId(), MakeHolder(ui64(buildInfo.ApplyTxId))); + } else { + buildInfo.ApplyTxId = {}; + buildInfo.ApplyTxStatus = NKikimrScheme::StatusSuccess; + buildInfo.ApplyTxDone = false; + + NIceDb::TNiceDb db(txc.DB); + Self->PersistBuildIndexApplyTxId(db, buildInfo); + Self->PersistBuildIndexApplyTxStatus(db, buildInfo); + Self->PersistBuildIndexApplyTxDone(db, buildInfo); + + ChangeState(BuildId, TIndexBuildInfo::EState::Filling); + Progress(BuildId); + } + break; case TIndexBuildInfo::EState::Applying: if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), ApplyPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1242,7 +1254,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Unlocking: if (buildInfo.UnlockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.UnlockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.UnlockTxDone) { @@ -1258,7 +1270,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Cancellation_Applying: if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), CancelPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1270,7 +1282,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Cancellation_Unlocking: if (buildInfo.UnlockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.UnlockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.UnlockTxDone) { @@ -1286,7 +1298,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Rejection_Applying: if (buildInfo.ApplyTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), CancelPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.ApplyTxDone) { @@ -1298,7 +1310,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil break; case TIndexBuildInfo::EState::Rejection_Unlocking: if (buildInfo.UnlockTxId == InvalidTxId) { - Send(Self->TxAllocatorClient, MakeHolder(), 0, ui64(BuildId)); + AllocateTxId(BuildId); } else if (buildInfo.UnlockTxStatus == NKikimrScheme::StatusSuccess) { Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId)); } else if (!buildInfo.UnlockTxDone) { @@ -1332,7 +1344,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil return TSerializedTableRange{{&from, 1}, false, {&to, 1}, true}; } - void InitiateShards(NIceDb::TNiceDb& db, TIndexBuildInfo& buildInfo) { + bool InitiateShards(NIceDb::TNiceDb& db, TIndexBuildInfo& buildInfo) { LOG_D("InitiateShards " << buildInfo.DebugString()); Y_ASSERT(buildInfo.Shards.empty()); @@ -1344,8 +1356,15 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil if (buildInfo.KMeans.Level == 1) { table = Self->Tables.at(buildInfo.TablePathId); } else { - auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName); - table = Self->Tables.at(path.Dive(buildInfo.KMeans.ReadFrom())->PathId); + auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom()); + table = Self->Tables.at(path->PathId); + + if (!path.IsLocked()) { // lock is needed to prevent table shards from being split + ChangeState(buildInfo.Id, TIndexBuildInfo::EState::LockBuild); + Progress(buildInfo.Id); + return false; + } + Y_ASSERT(path.LockedBy() == buildInfo.LockTxId); } auto tableColumns = NTableIndex::ExtractInfo(table); // skip dropped columns TSerializedTableRange shardRange = InfiniteRange(tableColumns.Keys.size()); @@ -1366,6 +1385,8 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Self->PersistBuildIndexUploadInitiate(db, BuildId, x.ShardIdx, it->second); } + + return true; } void DoComplete(const TActorContext& ctx) override { @@ -1483,10 +1504,11 @@ struct TSchemeShard::TIndexBuilder::TTxReplyRetry: public TSchemeShard::TIndexBu case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -1637,10 +1659,11 @@ struct TSchemeShard::TIndexBuilder::TTxReplySampleK: public TSchemeShard::TIndex case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -1762,10 +1785,11 @@ struct TSchemeShard::TIndexBuilder::TTxReplyLocalKMeans: public TSchemeShard::TI case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -1886,10 +1910,11 @@ struct TSchemeShard::TIndexBuilder::TTxReplyReshuffleKMeans: public TSchemeShard case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -2011,10 +2036,11 @@ struct TSchemeShard::TIndexBuilder::TTxReplyPrefixKMeans: public TSchemeShard::T case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -2093,10 +2119,11 @@ struct TSchemeShard::TIndexBuilder::TTxReplyUpload: public TSchemeShard::TIndexB case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -2246,10 +2273,11 @@ struct TSchemeShard::TIndexBuilder::TTxReplyProgress: public TSchemeShard::TInde case TIndexBuildInfo::EState::Initiating: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Unlocking: case TIndexBuildInfo::EState::Done: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Cancellation_Unlocking: case TIndexBuildInfo::EState::Cancelled: @@ -2325,6 +2353,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyCompleted: public TSchemeShard::TInd } case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Rejection_Applying: @@ -2351,7 +2380,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyCompleted: public TSchemeShard::TInd case TIndexBuildInfo::EState::Done: case TIndexBuildInfo::EState::Cancelled: case TIndexBuildInfo::EState::Rejected: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); } Progress(buildId); @@ -2432,7 +2461,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB if (statusCode != Ydb::StatusIds::SUCCESS) { buildInfo.Issue += TStringBuilder() - << "At " << Name(state) << " state got unsuccess propose result" + << "At " << state << " state got unsuccess propose result" << ", status: " << NKikimrScheme::EStatus_Name(record.GetStatus()) << ", reason: " << record.GetReason(); Self->PersistBuildIndexIssue(db, buildInfo); @@ -2451,7 +2480,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB // no op } else { buildInfo.Issue += TStringBuilder() - << "At " << Name(state) << " state got unsuccess propose result" + << "At " << state << " state got unsuccess propose result" << ", status: " << NKikimrScheme::EStatus_Name(record.GetStatus()) << ", reason: " << record.GetReason(); Self->PersistBuildIndexIssue(db, buildInfo); @@ -2492,6 +2521,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB } case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Rejection_Applying: { @@ -2549,7 +2579,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB case TIndexBuildInfo::EState::Done: case TIndexBuildInfo::EState::Cancelled: case TIndexBuildInfo::EState::Rejected: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); } Progress(buildId); @@ -2606,6 +2636,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyAllocate: public TSchemeShard::TInde break; case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Rejection_Applying: @@ -2628,7 +2659,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyAllocate: public TSchemeShard::TInde case TIndexBuildInfo::EState::Done: case TIndexBuildInfo::EState::Cancelled: case TIndexBuildInfo::EState::Rejected: - Y_FAIL_S("Unreachable " << Name(state)); + Y_FAIL_S("Unreachable " << state); } Progress(buildId); diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp index e07d231c38e3..9a3eecc50764 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp @@ -177,6 +177,11 @@ void TSchemeShard::TIndexBuilder::TTxBase::Send(TActorId dst, THolderTxAllocatorClient, MakeHolder(), 0, ui64(buildId)); +} + void TSchemeShard::TIndexBuilder::TTxBase::ChangeState(TIndexBuildId id, TIndexBuildInfo::EState state) { StateChanges.push_back(TChangeStateRec(id, state)); } @@ -215,6 +220,7 @@ void TSchemeShard::TIndexBuilder::TTxBase::Fill(NKikimrIndexBuilder::TIndexBuild case TIndexBuildInfo::EState::Filling: case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: + case TIndexBuildInfo::EState::LockBuild: index.SetState(Ydb::Table::IndexBuildState::STATE_TRANSFERING_DATA); index.SetProgress(indexInfo.CalcProgressPercent()); break; diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h index 1a70dd31ec5e..02f96b34aef3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h @@ -32,6 +32,7 @@ class TSchemeShard::TIndexBuilder::TTxBase: public NTabletFlatExecutor::TTransac protected: void Send(TActorId dst, THolder message, ui32 flags = 0, ui64 cookie = 0); + void AllocateTxId(TIndexBuildId buildId); void ChangeState(TIndexBuildId id, TIndexBuildInfo::EState state); void Progress(TIndexBuildId id); void Fill(NKikimrIndexBuilder::TIndexBuild& index, const TIndexBuildInfo& indexInfo); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 50f489a1e902..d6b90d7e3bc5 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3040,6 +3040,7 @@ struct TIndexBuildInfo: public TSimpleRefCount { Filling = 40, DropBuild = 45, CreateBuild = 46, + LockBuild = 47, Applying = 50, Unlocking = 60, Done = 200, diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index 01bae14754e7..c62f7be2d41f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -1875,6 +1875,10 @@ ui64 TPath::GetEffectiveACLVersion() const { return version; } +bool TPath::IsLocked() const { + return SS->LockedPaths.contains(Base()->PathId); +} + TTxId TPath::LockedBy() const { auto it = SS->LockedPaths.find(Base()->PathId); if (it != SS->LockedPaths.end()) { diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h index 4e167ac23dc3..071e488b7b86 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.h +++ b/ydb/core/tx/schemeshard/schemeshard_path.h @@ -183,6 +183,7 @@ class TPath { bool IsValidLeafName(TString& explain) const; TString GetEffectiveACL() const; ui64 GetEffectiveACLVersion() const; + bool IsLocked() const; TTxId LockedBy() const; bool IsActive() const;