Merge bf6f316 into 19153a4

azevaykin · web-flow · commit f7a4f78a4026 · 2024-07-19T05:17:17.000Z
diff --git a/ydb/core/base/table_index.cpp b/ydb/core/base/table_index.cpp
@@ -140,5 +140,9 @@ bool IsCompatibleIndex(const NKikimrSchemeOp::EIndexType indexType, const TTable
     return true;
 }
 
+bool IsImplTable(std::string_view tableName) {
+    return std::find(std::begin(ImplTables), std::end(ImplTables), tableName) != std::end(ImplTables);
+}
+
 }
 }
diff --git a/ydb/core/base/table_index.h b/ydb/core/base/table_index.h
@@ -21,8 +21,13 @@ struct TIndexColumns {
     TVector<TString> DataColumns;
 };
 
+inline constexpr const char* ImplTable = "indexImplTable";
+inline constexpr std::string_view ImplTables[] = {ImplTable, NTableVectorKmeansTreeIndex::LevelTable, NTableVectorKmeansTreeIndex::PostingTable};
+
 bool IsCompatibleIndex(const NKikimrSchemeOp::EIndexType indexType, const TTableColumns& table, const TIndexColumns& index, TString& explain);
 TTableColumns CalcTableImplDescription(const NKikimrSchemeOp::EIndexType indexType, const TTableColumns& table, const TIndexColumns& index);
 
+bool IsImplTable(std::string_view tableName);
+
 }
 }
diff --git a/ydb/core/grpc_services/rpc_describe_table.cpp b/ydb/core/grpc_services/rpc_describe_table.cpp
@@ -6,6 +6,7 @@
 
 #include "service_table.h"
 #include "rpc_common/rpc_common.h"
+#include <ydb/core/base/table_index.h>
 #include <ydb/core/tx/schemeshard/schemeshard.h>
 #include <ydb/core/ydb_convert/table_description.h>
 #include <ydb/core/ydb_convert/ydb_convert.h>
@@ -153,7 +154,7 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe
             record->MutableOptions()->SetReturnPartitionStats(true);
         }
 
-        if (AppData(ctx)->AllowPrivateTableDescribeForTest || path.EndsWith("/indexImplTable")) {
+        if (AppData(ctx)->AllowPrivateTableDescribeForTest || path.EndsWith(TStringBuilder() << "/" << NTableIndex::ImplTable)) {
             record->MutableOptions()->SetShowPrivateTable(true);
         }
 
diff --git a/ydb/core/kqp/gateway/utils/scheme_helpers.cpp b/ydb/core/kqp/gateway/utils/scheme_helpers.cpp
@@ -1,6 +1,7 @@
 #include "scheme_helpers.h"
 
 #include <ydb/core/base/path.h>
+#include <ydb/core/base/table_index.h>
 #include <ydb/core/protos/external_sources.pb.h>
 
 namespace NKikimr::NKqp::NSchemeHelpers {
@@ -46,7 +47,7 @@ bool SplitTablePath(const TString& tableName, const TString& database, std::pair
 }
 
 TString CreateIndexTablePath(const TString& tableName, const TString& indexName) {
-    return tableName + "/" + indexName + "/indexImplTable";
+    return tableName + "/" + indexName + "/" + NTableIndex::ImplTable;
 }
 
 bool SetDatabaseForLoginOperation(TString& result, bool getDomainLoginOnly, TMaybe<TString> domainName,
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp
@@ -52,17 +52,25 @@ TVector<ISubOperation::TPtr> ApplyBuildIndex(TOperationId nextId, const TTxTrans
 
     if (!indexName.empty())
     {
+        auto alterImplTableTransactionTemplate = [] (TPath index, TPath implIndexTable, TTableInfo::TPtr implIndexTableInfo) {
+            auto indexImplTableAltering = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpFinalizeBuildIndexImplTable);
+            auto alterTable = indexImplTableAltering.MutableAlterTable();
+            alterTable->SetName(implIndexTable.LeafName());
+            alterTable->MutablePartitionConfig()->MutableCompactionPolicy()->CopyFrom(implIndexTableInfo->PartitionConfig().GetCompactionPolicy());
+            alterTable->MutablePartitionConfig()->MutableCompactionPolicy()->SetKeepEraseMarkers(false);
+            alterTable->MutablePartitionConfig()->SetShadowData(false);
+            return indexImplTableAltering;
+        };
+
         TPath index = table.Child(indexName);
-        TPath implIndexTable = index.Child("indexImplTable");
-        TTableInfo::TPtr implIndexTableInfo = context.SS->Tables.at(implIndexTable.Base()->PathId);
-        auto indexImplTableAltering = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpFinalizeBuildIndexImplTable);
-        auto alterTable = indexImplTableAltering.MutableAlterTable();
-        alterTable->SetName(implIndexTable.LeafName());
-        alterTable->MutablePartitionConfig()->MutableCompactionPolicy()->CopyFrom(implIndexTableInfo->PartitionConfig().GetCompactionPolicy());
-        alterTable->MutablePartitionConfig()->MutableCompactionPolicy()->SetKeepEraseMarkers(false);
-        alterTable->MutablePartitionConfig()->SetShadowData(false);
-
-        result.push_back(CreateFinalizeBuildIndexImplTable(NextPartId(nextId, result), indexImplTableAltering));
+        for (const std::string_view implTable : NTableIndex::ImplTables) {
+            TPath implIndexTable = index.Child(implTable.data());
+            if (!implIndexTable.IsResolved()) {
+                continue;
+            }
+            TTableInfo::TPtr implIndexTableInfo = context.SS->Tables.at(implIndexTable.Base()->PathId);
+            result.push_back(CreateFinalizeBuildIndexImplTable(NextPartId(nextId, result), alterImplTableTransactionTemplate(index, implIndexTable, implIndexTableInfo)));
+        }
     }
 
     return result;
@@ -106,10 +114,10 @@ TVector<ISubOperation::TPtr> CancelBuildIndex(TOperationId nextId, const TTxTran
 
     if (!indexName.empty()) {
         TPath index = table.Child(indexName);
-        Y_ABORT_UNLESS(index.Base()->GetChildren().size() == 1);
+        Y_ABORT_UNLESS(index.Base()->GetChildren().size() >= 1);
         for (auto& indexChildItems: index.Base()->GetChildren()) {
             const TString& implTableName = indexChildItems.first;
-            Y_ABORT_UNLESS(implTableName == "indexImplTable", "unexpected name %s", implTableName.c_str());
+            Y_ABORT_UNLESS(NTableIndex::IsImplTable(implTableName), "unexpected name %s", implTableName.c_str());
 
             TPath implTable = index.Child(implTableName);
             {
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_part.cpp
@@ -152,10 +152,8 @@ ISubOperation::TPtr CascadeDropTableChildren(TVector<ISubOperation::TPtr>& resul
         }
 
         for (auto& [implName, implPathId] : child.Base()->GetChildren()) {
-            Y_ABORT_UNLESS(implName == "indexImplTable" 
+            Y_ABORT_UNLESS(NTableIndex::IsImplTable(implName) 
                         || implName == "streamImpl"
-                        || implName == NTableIndex::NTableVectorKmeansTreeIndex::LevelTable
-                        || implName == NTableIndex::NTableVectorKmeansTreeIndex::PostingTable
                 , "unexpected name %s", implName.c_str());
 
             TPath implPath = child.Child(implName);
diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp
@@ -226,9 +226,13 @@ class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder
             case Ydb::Table::TableIndex::TypeCase::kGlobalUniqueIndex:
                 explain = "unsupported index type to build";
                 return false;
-            case Ydb::Table::TableIndex::TypeCase::kGlobalVectorKmeansTreeIndex:
-                explain = "unsupported vector index type to build";
-                return false;
+            case Ydb::Table::TableIndex::TypeCase::kGlobalVectorKmeansTreeIndex: {
+                buildInfo->IndexType = NKikimrSchemeOp::EIndexType::EIndexTypeGlobalVectorKmeansTree;
+                NKikimrSchemeOp::TVectorIndexKmeansTreeDescription vectorIndexKmeansTreeDescription;
+                *vectorIndexKmeansTreeDescription.MutableSettings() = index.global_vector_kmeans_tree_index().vector_settings();
+                buildInfo->SpecializedIndexDescription = vectorIndexKmeansTreeDescription;
+                break;
+            }
             case Ydb::Table::TableIndex::TypeCase::TYPE_NOT_SET:
                 explain = "invalid or unset index type";
                 return false;
diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp
@@ -253,6 +253,13 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil
             } else if (!buildInfo->InitiateTxDone) {
                 Send(Self->SelfId(), MakeHolder<TEvSchemeShard::TEvNotifyTxCompletion>(ui64(buildInfo->InitiateTxId)));
             } else {
+                // TODO add vector index filling
+                if (buildInfo->IndexType == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalVectorKmeansTree) {
+                    ChangeState(BuildId, TIndexBuildInfo::EState::Applying);
+                    Progress(BuildId);
+                    break;
+                }
+
                 ChangeState(BuildId, TIndexBuildInfo::EState::Filling);
                 Progress(BuildId);
             }
@@ -314,7 +321,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil
             }
 
             if (buildInfo->ImplTablePath.Empty() && buildInfo->IsBuildIndex()) {
-                TPath implTable = TPath::Init(buildInfo->TablePathId, Self).Dive(buildInfo->IndexName).Dive("indexImplTable");
+                TPath implTable = TPath::Init(buildInfo->TablePathId, Self).Dive(buildInfo->IndexName).Dive(NTableIndex::ImplTable);
                 buildInfo->ImplTablePath = implTable.PathString();
 
                 TTableInfo::TPtr implTableInfo = Self->Tables.at(implTable.Base()->PathId);
diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp
@@ -2122,6 +2122,10 @@ void TIndexBuildInfo::SerializeToProto(TSchemeShard* ss, NKikimrSchemeOp::TIndex
     for (const auto& implTableDescription : ImplTableDescriptions) {
         *index.AddIndexImplTableDescriptions() = implTableDescription;
     }
+
+    if (IndexType == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalVectorKmeansTree) {
+        *index.MutableVectorIndexKmeansTreeDescription() = std::get<NKikimrSchemeOp::TVectorIndexKmeansTreeDescription>(SpecializedIndexDescription);
+    }
 }
 
 void TIndexBuildInfo::SerializeToProto(TSchemeShard* ss, NKikimrIndexBuilder::TColumnBuildSettings* result) const {
diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h
@@ -2927,6 +2927,8 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
     NTableIndex::TTableColumns ImplTableColumns;
     TVector<NKikimrSchemeOp::TTableDescription> ImplTableDescriptions;
 
+    std::variant<std::monostate, NKikimrSchemeOp::TVectorIndexKmeansTreeDescription> SpecializedIndexDescription;
+
     EState State = EState::Invalid;
     TString Issue;
 
diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.cpp b/ydb/core/tx/schemeshard/schemeshard_utils.cpp
@@ -451,7 +451,7 @@ NKikimrSchemeOp::TTableDescription CalcImplTableDesc(
 {
     NKikimrSchemeOp::TTableDescription implTableDesc;
 
-    implTableDesc.SetName("indexImplTable");
+    implTableDesc.SetName(NTableIndex::ImplTable);
 
     SetImplTablePartitionConfig(baseTableInfo->PartitionConfig(), indexTableDesc, implTableDesc);
 
@@ -467,7 +467,7 @@ NKikimrSchemeOp::TTableDescription CalcImplTableDesc(
 {
     NKikimrSchemeOp::TTableDescription implTableDesc;
 
-    implTableDesc.SetName("indexImplTable");
+    implTableDesc.SetName(NTableIndex::ImplTable);
 
     SetImplTablePartitionConfig(baseTableDescr.GetPartitionConfig(), indexTableDesc, implTableDesc);
 

Original file line number	Diff line number	Diff line change
`@@ -140,5 +140,9 @@ bool IsCompatibleIndex(const NKikimrSchemeOp::EIndexType indexType, const TTable`
`140`	`140`	`return true;`
`141`	`141`	`}`
`142`	`142`
	`143`	`+bool IsImplTable(std::string_view tableName) {`
	`144`	`+ return std::find(std::begin(ImplTables), std::end(ImplTables), tableName) != std::end(ImplTables);`
	`145`	`+}`
	`146`	`+`
`143`	`147`	`}`
`144`	`148`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,8 +21,13 @@ struct TIndexColumns {`
`21`	`21`	`TVector<TString> DataColumns;`
`22`	`22`	`};`
`23`	`23`
	`24`	`+inline constexpr const char* ImplTable = "indexImplTable";`
	`25`	`+inline constexpr std::string_view ImplTables[] = {ImplTable, NTableVectorKmeansTreeIndex::LevelTable, NTableVectorKmeansTreeIndex::PostingTable};`
	`26`	`+`
`24`	`27`	`bool IsCompatibleIndex(const NKikimrSchemeOp::EIndexType indexType, const TTableColumns& table, const TIndexColumns& index, TString& explain);`
`25`	`28`	`TTableColumns CalcTableImplDescription(const NKikimrSchemeOp::EIndexType indexType, const TTableColumns& table, const TIndexColumns& index);`
`26`	`29`
	`30`	`+bool IsImplTable(std::string_view tableName);`
	`31`	`+`
`27`	`32`	`}`
`28`	`33`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,7 @@`
`6`	`6`
`7`	`7`	`#include "service_table.h"`
`8`	`8`	`#include "rpc_common/rpc_common.h"`
	`9`	`+#include <ydb/core/base/table_index.h>`
`9`	`10`	`#include <ydb/core/tx/schemeshard/schemeshard.h>`
`10`	`11`	`#include <ydb/core/ydb_convert/table_description.h>`
`11`	`12`	`#include <ydb/core/ydb_convert/ydb_convert.h>`
`@@ -153,7 +154,7 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe`
`153`	`154`	`record->MutableOptions()->SetReturnPartitionStats(true);`
`154`	`155`	`}`
`155`	`156`
`156`		`- if (AppData(ctx)->AllowPrivateTableDescribeForTest \|\| path.EndsWith("/indexImplTable")) {`
	`157`	`+ if (AppData(ctx)->AllowPrivateTableDescribeForTest \|\| path.EndsWith(TStringBuilder() << "/" << NTableIndex::ImplTable)) {`
`157`	`158`	`record->MutableOptions()->SetShowPrivateTable(true);`
`158`	`159`	`}`
`159`	`160`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`#include "scheme_helpers.h"`
`2`	`2`
`3`	`3`	`#include <ydb/core/base/path.h>`
	`4`	`+#include <ydb/core/base/table_index.h>`
`4`	`5`	`#include <ydb/core/protos/external_sources.pb.h>`
`5`	`6`
`6`	`7`	`namespace NKikimr::NKqp::NSchemeHelpers {`
`@@ -46,7 +47,7 @@ bool SplitTablePath(const TString& tableName, const TString& database, std::pair`
`46`	`47`	`}`
`47`	`48`
`48`	`49`	`TString CreateIndexTablePath(const TString& tableName, const TString& indexName) {`
`49`		`- return tableName + "/" + indexName + "/indexImplTable";`
	`50`	`+ return tableName + "/" + indexName + "/" + NTableIndex::ImplTable;`
`50`	`51`	`}`
`51`	`52`
`52`	`53`	`bool SetDatabaseForLoginOperation(TString& result, bool getDomainLoginOnly, TMaybe<TString> domainName,`
Original file line number	Diff line number	Diff line change
`@@ -2122,6 +2122,10 @@ void TIndexBuildInfo::SerializeToProto(TSchemeShard* ss, NKikimrSchemeOp::TIndex`
`2122`	`2122`	`for (const auto& implTableDescription : ImplTableDescriptions) {`
`2123`	`2123`	`*index.AddIndexImplTableDescriptions() = implTableDescription;`
`2124`	`2124`	`}`
	`2125`	`+`
	`2126`	`+ if (IndexType == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalVectorKmeansTree) {`
	`2127`	`+ *index.MutableVectorIndexKmeansTreeDescription() = std::get<NKikimrSchemeOp::TVectorIndexKmeansTreeDescription>(SpecializedIndexDescription);`
	`2128`	`+ }`
`2125`	`2129`	`}`
`2126`	`2130`
`2127`	`2131`	`void TIndexBuildInfo::SerializeToProto(TSchemeShard* ss, NKikimrIndexBuilder::TColumnBuildSettings* result) const {`
Original file line number	Diff line number	Diff line change
`@@ -451,7 +451,7 @@ NKikimrSchemeOp::TTableDescription CalcImplTableDesc(`
`451`	`451`	`{`
`452`	`452`	`NKikimrSchemeOp::TTableDescription implTableDesc;`
`453`	`453`
`454`		`- implTableDesc.SetName("indexImplTable");`
	`454`	`+ implTableDesc.SetName(NTableIndex::ImplTable);`
`455`	`455`
`456`	`456`	`SetImplTablePartitionConfig(baseTableInfo->PartitionConfig(), indexTableDesc, implTableDesc);`
`457`	`457`
`@@ -467,7 +467,7 @@ NKikimrSchemeOp::TTableDescription CalcImplTableDesc(`
`467`	`467`	`{`
`468`	`468`	`NKikimrSchemeOp::TTableDescription implTableDesc;`
`469`	`469`
`470`		`- implTableDesc.SetName("indexImplTable");`
	`470`	`+ implTableDesc.SetName(NTableIndex::ImplTable);`
`471`	`471`
`472`	`472`	`SetImplTablePartitionConfig(baseTableDescr.GetPartitionConfig(), indexTableDesc, implTableDesc);`
`473`	`473`