Skip to content

Commit d1adfd1

Browse files
jepett0jepett0
authored
Ingore basic scheme limits for internal operations (#14813)
1 parent cb5a9d0 commit d1adfd1

7 files changed

+154
-17
lines changed

‎ydb/core/protos/index_builder.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ message TEvCreateRequest {
4949
optional uint64 TxId = 2;
5050
optional string DatabaseName = 3;
5151
optional TIndexBuildSettings Settings = 4;
52+
// Internal flag is true for system-generated operations and is false for those initiated directly by the user.
53+
optional bool Internal = 5 [default = false];
5254
}
5355

5456
message TEvCreateResponse {

‎ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ TVector<ISubOperation::TPtr> CreateBuildColumn(TOperationId opId, const TTxTrans
2424
{
2525
auto outTx = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable);
2626
*outTx.MutableLockGuard() = tx.GetLockGuard();
27+
outTx.SetInternal(tx.GetInternal());
2728

2829
auto& snapshot = *outTx.MutableInitiateBuildIndexMainTable();
2930
snapshot.SetTableName(table.LeafName());
@@ -62,8 +63,12 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
6263
checks
6364
.IsValidLeafName()
6465
.PathsLimit(2) // index and impl-table
65-
.DirChildrenLimit()
66-
.ShardsLimit(1); // impl-table
66+
.DirChildrenLimit();
67+
68+
if (!tx.GetInternal()) {
69+
checks
70+
.ShardsLimit(1); // impl-table
71+
}
6772

6873
if (!checks) {
6974
return {CreateReject(opId, checks.GetStatus(), checks.GetError())};
@@ -95,6 +100,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
95100
*outTx.MutableLockGuard() = tx.GetLockGuard();
96101
outTx.MutableCreateTableIndex()->CopyFrom(indexDesc);
97102
outTx.MutableCreateTableIndex()->SetState(NKikimrSchemeOp::EIndexStateWriteOnly);
103+
outTx.SetInternal(tx.GetInternal());
98104

99105
if (!indexDesc.HasType()) {
100106
outTx.MutableCreateTableIndex()->SetType(NKikimrSchemeOp::EIndexTypeGlobal);
@@ -106,6 +112,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
106112
{
107113
auto outTx = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable);
108114
*outTx.MutableLockGuard() = tx.GetLockGuard();
115+
outTx.SetInternal(tx.GetInternal());
109116

110117
auto& snapshot = *outTx.MutableInitiateBuildIndexMainTable();
111118
snapshot.SetTableName(table.LeafName());
@@ -118,6 +125,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
118125

119126
auto outTx = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexImplTable);
120127
*outTx.MutableCreateTable() = std::move(implTableDesc);
128+
outTx.SetInternal(tx.GetInternal());
121129

122130
return CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx);
123131
};

‎ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,13 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
110110
{
111111
auto checks = baseTablePath.Check();
112112
checks
113-
.PathShardsLimit(baseShards)
114-
.PathsLimit(pathToCreate)
115-
.ShardsLimit(shardsToCreate);
113+
.PathsLimit(pathToCreate);
114+
115+
if (!tx.GetInternal()) {
116+
checks
117+
.PathShardsLimit(baseShards)
118+
.ShardsLimit(shardsToCreate);
119+
}
116120

117121
if (!checks) {
118122
return {CreateReject(nextId, NKikimrScheme::EStatus::StatusResourceExhausted, checks.GetError())};
@@ -224,6 +228,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
224228
auto scheme = TransactionTemplate(tx.GetWorkingDir(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable);
225229
scheme.SetFailOnExist(tx.GetFailOnExist());
226230
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
231+
scheme.SetInternal(tx.GetInternal());
227232

228233
scheme.MutableCreateTable()->CopyFrom(baseTableDescription);
229234
if (tx.HasAlterUserAttributes()) {
@@ -270,6 +275,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
270275
NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable);
271276
scheme.SetFailOnExist(tx.GetFailOnExist());
272277
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
278+
scheme.SetInternal(tx.GetInternal());
273279

274280
*scheme.MutableCreateTable() = std::move(implTableDesc);
275281

@@ -311,6 +317,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
311317
NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence);
312318
scheme.SetFailOnExist(tx.GetFailOnExist());
313319
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
320+
scheme.SetInternal(tx.GetInternal());
314321

315322
*scheme.MutableSequence() = sequenceDescription;
316323

‎ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -503,9 +503,13 @@ class TCreateTable: public TSubOperation {
503503
.IsValidLeafName()
504504
.PathsLimit()
505505
.DirChildrenLimit()
506-
.ShardsLimit(shardsToCreate)
507-
.PathShardsLimit(shardsToCreate)
508506
.IsValidACL(acl);
507+
508+
if (!Transaction.GetInternal()) {
509+
checks
510+
.ShardsLimit(shardsToCreate)
511+
.PathShardsLimit(shardsToCreate);
512+
}
509513
}
510514

511515
if (!checks) {

‎ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,12 @@ class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder
117117
checks
118118
.IsValidLeafName()
119119
.PathsLimit(2) // index and impl-table
120-
.DirChildrenLimit()
121-
.ShardsLimit(1); // impl-table
120+
.DirChildrenLimit();
121+
122+
if (!request.GetInternal()) {
123+
checks
124+
.ShardsLimit(1); // impl-table
125+
}
122126

123127
if (!checks) {
124128
return Reply(checks.GetStatus(), checks.GetError());

‎ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,9 @@ THolder<TEvIndexBuilder::TEvCreateRequest> BuildIndexPropose(
217217

218218
const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss);
219219
auto propose = MakeHolder<TEvIndexBuilder::TEvCreateRequest>(ui64(txId), domainPath.PathString(), std::move(settings));
220-
(*propose->Record.MutableOperationParams()->mutable_labels())["uid"] = uid;
220+
auto& request = propose->Record;
221+
(*request.MutableOperationParams()->mutable_labels())["uid"] = uid;
222+
request.SetInternal(true);
221223

222224
return propose;
223225
}
@@ -259,11 +261,11 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateChangefeedPropose(
259261
if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) {
260262
return nullptr;
261263
}
262-
264+
263265
if (topic.has_retention_period()) {
264266
cdcStream.SetRetentionPeriodSeconds(topic.retention_period().seconds());
265267
}
266-
268+
267269
if (topic.has_partitioning_settings()) {
268270
i64 minActivePartitions =
269271
topic.partitioning_settings().min_active_partitions();

‎ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp

Lines changed: 115 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,7 @@ namespace {
296296
case EPathTypeView:
297297
result.emplace(prefix + "/create_view.sql", item.CreationQuery);
298298
break;
299-
case EPathTypeCdcStream:
299+
case EPathTypeCdcStream:
300300
result.emplace(prefix + "/changefeed_description.pb", item.Changefeed.Changefeed);
301301
result.emplace(prefix + "/topic_description.pb", item.Changefeed.Topic);
302302
break;
@@ -5031,6 +5031,116 @@ Y_UNIT_TEST_SUITE(TImportTests) {
50315031
Y_UNIT_TEST(ChangefeedsWithTablePermissions) {
50325032
TestImportChangefeeds(3, AddedSchemeWithPermissions);
50335033
}
5034+
5035+
Y_UNIT_TEST(IgnoreBasicSchemeLimits) {
5036+
TTestBasicRuntime runtime;
5037+
TTestEnv env(runtime);
5038+
ui64 txId = 100;
5039+
5040+
TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", R"(
5041+
Name: "Alice"
5042+
)");
5043+
TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", R"(
5044+
Name: "Alice"
5045+
ExternalSchemeShard: true
5046+
PlanResolution: 50
5047+
Coordinators: 1
5048+
Mediators: 1
5049+
TimeCastBucketsPerMediator: 2
5050+
StoragePools {
5051+
Name: "Alice:hdd"
5052+
Kind: "hdd"
5053+
}
5054+
)");
5055+
env.TestWaitNotification(runtime, txId);
5056+
5057+
ui64 tenantSchemeShard = 0;
5058+
TestDescribeResult(DescribePath(runtime, "/MyRoot/Alice"), {
5059+
NLs::ExtractTenantSchemeshard(&tenantSchemeShard)
5060+
});
5061+
UNIT_ASSERT_UNEQUAL(tenantSchemeShard, 0);
5062+
5063+
TSchemeLimits basicLimits;
5064+
basicLimits.MaxShards = 4;
5065+
basicLimits.MaxShardsInPath = 2;
5066+
SetSchemeshardSchemaLimits(runtime, basicLimits, tenantSchemeShard);
5067+
5068+
TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/Alice"), {
5069+
NLs::DomainLimitsIs(basicLimits.MaxPaths, basicLimits.MaxShards),
5070+
NLs::PathsInsideDomain(0),
5071+
NLs::ShardsInsideDomain(3)
5072+
});
5073+
5074+
TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/Alice", R"(
5075+
Name: "table1"
5076+
Columns { Name: "Key" Type: "Uint64" }
5077+
Columns { Name: "Value" Type: "Utf8" }
5078+
KeyColumnNames: ["Key"]
5079+
)");
5080+
env.TestWaitNotification(runtime, txId, tenantSchemeShard);
5081+
5082+
TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/Alice", R"(
5083+
Name: "table2"
5084+
Columns { Name: "Key" Type: "Uint64" }
5085+
Columns { Name: "Value" Type: "Utf8" }
5086+
KeyColumnNames: ["Key"]
5087+
)",
5088+
{ NKikimrScheme::StatusResourceExhausted }
5089+
);
5090+
5091+
const auto data = GenerateTestData(R"(
5092+
columns {
5093+
name: "key"
5094+
type { optional_type { item { type_id: UTF8 } } }
5095+
}
5096+
columns {
5097+
name: "value"
5098+
type { optional_type { item { type_id: UTF8 } } }
5099+
}
5100+
primary_key: "key"
5101+
partition_at_keys {
5102+
split_points {
5103+
type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } }
5104+
value { items { text_value: "b" } }
5105+
}
5106+
}
5107+
indexes {
5108+
name: "ByValue"
5109+
index_columns: "value"
5110+
global_index {}
5111+
}
5112+
)",
5113+
{{"a", 1}, {"b", 1}}
5114+
);
5115+
5116+
TPortManager portManager;
5117+
const ui16 port = portManager.GetPort();
5118+
5119+
TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port));
5120+
UNIT_ASSERT(s3Mock.Start());
5121+
5122+
const auto importId = ++txId;
5123+
TestImport(runtime, tenantSchemeShard, importId, "/MyRoot/Alice", Sprintf(R"(
5124+
ImportFromS3Settings {
5125+
endpoint: "localhost:%d"
5126+
scheme: HTTP
5127+
items {
5128+
source_prefix: ""
5129+
destination_path: "/MyRoot/Alice/ImportDir/Table"
5130+
}
5131+
}
5132+
)",
5133+
port
5134+
));
5135+
env.TestWaitNotification(runtime, importId, tenantSchemeShard);
5136+
TestGetImport(runtime, tenantSchemeShard, importId, "/MyRoot/Alice");
5137+
5138+
TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/Alice"), {
5139+
NLs::DomainLimitsIs(basicLimits.MaxPaths, basicLimits.MaxShards),
5140+
NLs::PathsInsideDomain(5),
5141+
NLs::ShardsInsideDomain(7)
5142+
});
5143+
}
50345144
}
50355145

50365146
Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
@@ -5380,7 +5490,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
53805490
const auto changefeedName = "update_changefeed";
53815491

53825492
schemes.emplace("", R"(
5383-
columns {
5493+
columns {
53845494
name: "key"
53855495
type { optional_type { item { type_id: UTF8 } } }
53865496
}
@@ -5447,11 +5557,11 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
54475557
}
54485558
}
54495559
)";
5450-
5560+
54515561
NAttr::TAttributes attr;
54525562
attr.emplace(NAttr::EKeys::TOPIC_DESCRIPTION, topicDesc);
54535563

5454-
schemes.emplace("/update_feed",
5564+
schemes.emplace("/update_feed",
54555565
TTypedScheme {
54565566
EPathTypeCdcStream,
54575567
changefeedDesc,
@@ -5468,7 +5578,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
54685578
Y_UNIT_TEST(CancelShouldSucceedOnSingleChangefeed) {
54695579
CancelShouldSucceed(GetSchemeWithChangefeed());
54705580
}
5471-
5581+
54725582
Y_UNIT_TEST(CancelShouldSucceedOnDependentView) {
54735583
CancelShouldSucceed(
54745584
{

0 commit comments

Comments
 (0)