Skip to content

Commit 9384dd8

Browse files
jepett0jepett0
authored
25-1-1: Ingore basic scheme limits for internal operations (#16001)
1 parent 7febcbd commit 9384dd8

7 files changed

+153
-16
lines changed

‎ydb/core/protos/index_builder.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ message TEvCreateRequest {
4949
optional uint64 TxId = 2;
5050
optional string DatabaseName = 3;
5151
optional TIndexBuildSettings Settings = 4;
52+
// Internal flag is true for system-generated operations and is false for those initiated directly by the user.
53+
optional bool Internal = 5 [default = false];
5254
}
5355

5456
message TEvCreateResponse {

‎ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ TVector<ISubOperation::TPtr> CreateBuildColumn(TOperationId opId, const TTxTrans
2424
{
2525
auto outTx = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable);
2626
*outTx.MutableLockGuard() = tx.GetLockGuard();
27+
outTx.SetInternal(tx.GetInternal());
2728

2829
auto& snapshot = *outTx.MutableInitiateBuildIndexMainTable();
2930
snapshot.SetTableName(table.LeafName());
@@ -62,8 +63,12 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
6263
checks
6364
.IsValidLeafName()
6465
.PathsLimit(2) // index and impl-table
65-
.DirChildrenLimit()
66-
.ShardsLimit(1); // impl-table
66+
.DirChildrenLimit();
67+
68+
if (!tx.GetInternal()) {
69+
checks
70+
.ShardsLimit(1); // impl-table
71+
}
6772

6873
if (!checks) {
6974
return {CreateReject(opId, checks.GetStatus(), checks.GetError())};
@@ -95,6 +100,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
95100
*outTx.MutableLockGuard() = tx.GetLockGuard();
96101
outTx.MutableCreateTableIndex()->CopyFrom(indexDesc);
97102
outTx.MutableCreateTableIndex()->SetState(NKikimrSchemeOp::EIndexStateWriteOnly);
103+
outTx.SetInternal(tx.GetInternal());
98104

99105
if (!indexDesc.HasType()) {
100106
outTx.MutableCreateTableIndex()->SetType(NKikimrSchemeOp::EIndexTypeGlobal);
@@ -106,6 +112,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
106112
{
107113
auto outTx = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable);
108114
*outTx.MutableLockGuard() = tx.GetLockGuard();
115+
outTx.SetInternal(tx.GetInternal());
109116

110117
auto& snapshot = *outTx.MutableInitiateBuildIndexMainTable();
111118
snapshot.SetTableName(table.LeafName());
@@ -118,6 +125,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
118125

119126
auto outTx = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexImplTable);
120127
*outTx.MutableCreateTable() = std::move(implTableDesc);
128+
outTx.SetInternal(tx.GetInternal());
121129

122130
return CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx);
123131
};

‎ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,13 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
110110
{
111111
auto checks = baseTablePath.Check();
112112
checks
113-
.PathShardsLimit(baseShards)
114-
.PathsLimit(pathToCreate)
115-
.ShardsLimit(shardsToCreate);
113+
.PathsLimit(pathToCreate);
114+
115+
if (!tx.GetInternal()) {
116+
checks
117+
.PathShardsLimit(baseShards)
118+
.ShardsLimit(shardsToCreate);
119+
}
116120

117121
if (!checks) {
118122
return {CreateReject(nextId, NKikimrScheme::EStatus::StatusResourceExhausted, checks.GetError())};
@@ -224,6 +228,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
224228
auto scheme = TransactionTemplate(tx.GetWorkingDir(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable);
225229
scheme.SetFailOnExist(tx.GetFailOnExist());
226230
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
231+
scheme.SetInternal(tx.GetInternal());
227232

228233
scheme.MutableCreateTable()->CopyFrom(baseTableDescription);
229234
if (tx.HasAlterUserAttributes()) {
@@ -270,6 +275,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
270275
NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable);
271276
scheme.SetFailOnExist(tx.GetFailOnExist());
272277
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
278+
scheme.SetInternal(tx.GetInternal());
273279

274280
*scheme.MutableCreateTable() = std::move(implTableDesc);
275281

@@ -311,6 +317,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
311317
NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence);
312318
scheme.SetFailOnExist(tx.GetFailOnExist());
313319
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
320+
scheme.SetInternal(tx.GetInternal());
314321

315322
*scheme.MutableSequence() = sequenceDescription;
316323

‎ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -503,9 +503,13 @@ class TCreateTable: public TSubOperation {
503503
.IsValidLeafName()
504504
.PathsLimit()
505505
.DirChildrenLimit()
506-
.ShardsLimit(shardsToCreate)
507-
.PathShardsLimit(shardsToCreate)
508506
.IsValidACL(acl);
507+
508+
if (!Transaction.GetInternal()) {
509+
checks
510+
.ShardsLimit(shardsToCreate)
511+
.PathShardsLimit(shardsToCreate);
512+
}
509513
}
510514

511515
if (!checks) {

‎ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,12 @@ class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder
117117
checks
118118
.IsValidLeafName()
119119
.PathsLimit(2) // index and impl-table
120-
.DirChildrenLimit()
121-
.ShardsLimit(1); // impl-table
120+
.DirChildrenLimit();
121+
122+
if (!request.GetInternal()) {
123+
checks
124+
.ShardsLimit(1); // impl-table
125+
}
122126

123127
if (!checks) {
124128
return Reply(checks.GetStatus(), checks.GetError());

‎ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,9 @@ THolder<TEvIndexBuilder::TEvCreateRequest> BuildIndexPropose(
220220

221221
const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss);
222222
auto propose = MakeHolder<TEvIndexBuilder::TEvCreateRequest>(ui64(txId), domainPath.PathString(), std::move(settings));
223-
(*propose->Record.MutableOperationParams()->mutable_labels())["uid"] = uid;
223+
auto& request = propose->Record;
224+
(*request.MutableOperationParams()->mutable_labels())["uid"] = uid;
225+
request.SetInternal(true);
224226

225227
return propose;
226228
}
@@ -262,11 +264,11 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateChangefeedPropose(
262264
if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) {
263265
return nullptr;
264266
}
265-
267+
266268
if (topic.has_retention_period()) {
267269
cdcStream.SetRetentionPeriodSeconds(topic.retention_period().seconds());
268270
}
269-
271+
270272
if (topic.has_partitioning_settings()) {
271273
i64 minActivePartitions =
272274
topic.partitioning_settings().min_active_partitions();

‎ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp

Lines changed: 114 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5310,6 +5310,116 @@ Y_UNIT_TEST_SUITE(TImportTests) {
53105310
Y_UNIT_TEST(ChangefeedsWithTablePermissions) {
53115311
TestImportChangefeeds(3, AddedSchemeWithPermissions);
53125312
}
5313+
5314+
Y_UNIT_TEST(IgnoreBasicSchemeLimits) {
5315+
TTestBasicRuntime runtime;
5316+
TTestEnv env(runtime);
5317+
ui64 txId = 100;
5318+
5319+
TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", R"(
5320+
Name: "Alice"
5321+
)");
5322+
TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", R"(
5323+
Name: "Alice"
5324+
ExternalSchemeShard: true
5325+
PlanResolution: 50
5326+
Coordinators: 1
5327+
Mediators: 1
5328+
TimeCastBucketsPerMediator: 2
5329+
StoragePools {
5330+
Name: "Alice:hdd"
5331+
Kind: "hdd"
5332+
}
5333+
)");
5334+
env.TestWaitNotification(runtime, txId);
5335+
5336+
ui64 tenantSchemeShard = 0;
5337+
TestDescribeResult(DescribePath(runtime, "/MyRoot/Alice"), {
5338+
NLs::ExtractTenantSchemeshard(&tenantSchemeShard)
5339+
});
5340+
UNIT_ASSERT_UNEQUAL(tenantSchemeShard, 0);
5341+
5342+
TSchemeLimits basicLimits;
5343+
basicLimits.MaxShards = 4;
5344+
basicLimits.MaxShardsInPath = 2;
5345+
SetSchemeshardSchemaLimits(runtime, basicLimits, tenantSchemeShard);
5346+
5347+
TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/Alice"), {
5348+
NLs::DomainLimitsIs(basicLimits.MaxPaths, basicLimits.MaxShards),
5349+
NLs::PathsInsideDomain(0),
5350+
NLs::ShardsInsideDomain(3)
5351+
});
5352+
5353+
TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/Alice", R"(
5354+
Name: "table1"
5355+
Columns { Name: "Key" Type: "Uint64" }
5356+
Columns { Name: "Value" Type: "Utf8" }
5357+
KeyColumnNames: ["Key"]
5358+
)");
5359+
env.TestWaitNotification(runtime, txId, tenantSchemeShard);
5360+
5361+
TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/Alice", R"(
5362+
Name: "table2"
5363+
Columns { Name: "Key" Type: "Uint64" }
5364+
Columns { Name: "Value" Type: "Utf8" }
5365+
KeyColumnNames: ["Key"]
5366+
)",
5367+
{ NKikimrScheme::StatusResourceExhausted }
5368+
);
5369+
5370+
const auto data = GenerateTestData(R"(
5371+
columns {
5372+
name: "key"
5373+
type { optional_type { item { type_id: UTF8 } } }
5374+
}
5375+
columns {
5376+
name: "value"
5377+
type { optional_type { item { type_id: UTF8 } } }
5378+
}
5379+
primary_key: "key"
5380+
partition_at_keys {
5381+
split_points {
5382+
type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } }
5383+
value { items { text_value: "b" } }
5384+
}
5385+
}
5386+
indexes {
5387+
name: "ByValue"
5388+
index_columns: "value"
5389+
global_index {}
5390+
}
5391+
)",
5392+
{{"a", 1}, {"b", 1}}
5393+
);
5394+
5395+
TPortManager portManager;
5396+
const ui16 port = portManager.GetPort();
5397+
5398+
TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port));
5399+
UNIT_ASSERT(s3Mock.Start());
5400+
5401+
const auto importId = ++txId;
5402+
TestImport(runtime, tenantSchemeShard, importId, "/MyRoot/Alice", Sprintf(R"(
5403+
ImportFromS3Settings {
5404+
endpoint: "localhost:%d"
5405+
scheme: HTTP
5406+
items {
5407+
source_prefix: ""
5408+
destination_path: "/MyRoot/Alice/ImportDir/Table"
5409+
}
5410+
}
5411+
)",
5412+
port
5413+
));
5414+
env.TestWaitNotification(runtime, importId, tenantSchemeShard);
5415+
TestGetImport(runtime, tenantSchemeShard, importId, "/MyRoot/Alice");
5416+
5417+
TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/Alice"), {
5418+
NLs::DomainLimitsIs(basicLimits.MaxPaths, basicLimits.MaxShards),
5419+
NLs::PathsInsideDomain(5),
5420+
NLs::ShardsInsideDomain(7)
5421+
});
5422+
}
53135423
}
53145424

53155425
Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
@@ -5701,7 +5811,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
57015811
const auto changefeedName = "update_changefeed";
57025812

57035813
schemes.emplace("", R"(
5704-
columns {
5814+
columns {
57055815
name: "key"
57065816
type { optional_type { item { type_id: UTF8 } } }
57075817
}
@@ -5768,11 +5878,11 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
57685878
}
57695879
}
57705880
)";
5771-
5881+
57725882
NAttr::TAttributes attr;
57735883
attr.emplace(NAttr::EKeys::TOPIC_DESCRIPTION, topicDesc);
57745884

5775-
schemes.emplace("/update_feed",
5885+
schemes.emplace("/update_feed",
57765886
TTypedScheme {
57775887
EPathTypeCdcStream,
57785888
changefeedDesc,
@@ -5789,7 +5899,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
57895899
Y_UNIT_TEST(CancelShouldSucceedOnSingleChangefeed) {
57905900
CancelShouldSucceed(GetSchemeWithChangefeed());
57915901
}
5792-
5902+
57935903
Y_UNIT_TEST(CancelShouldSucceedOnDependentView) {
57945904
CancelShouldSucceed(
57955905
{

0 commit comments

Comments
 (0)