Skip to content

Commit 0f92844

Browse files
jepett0jepett0
authored
Merge b48b108 into f7d665b 
2 parents f7d665b + b48b108 commit 0f92844

7 files changed

+149
-11
lines changed

‎ydb/core/protos/index_builder.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ message TEvCreateRequest {
4949
optional uint64 TxId = 2;
5050
optional string DatabaseName = 3;
5151
optional TIndexBuildSettings Settings = 4;
52+
// Internal flag is true for system-generated operations and is false for those initiated directly by the user.
53+
optional bool Internal = 5 [default = false];
5254
}
5355

5456
message TEvCreateResponse {

‎ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ TVector<ISubOperation::TPtr> CreateBuildColumn(TOperationId opId, const TTxTrans
2222
{
2323
auto outTx = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable);
2424
*outTx.MutableLockGuard() = tx.GetLockGuard();
25+
outTx.SetInternal(tx.GetInternal());
2526

2627
auto& snapshot = *outTx.MutableInitiateBuildIndexMainTable();
2728
snapshot.SetTableName(table.LeafName());
@@ -60,8 +61,12 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
6061
checks
6162
.IsValidLeafName()
6263
.PathsLimit(2) // index and impl-table
63-
.DirChildrenLimit()
64-
.ShardsLimit(1); // impl-table
64+
.DirChildrenLimit();
65+
66+
if (!tx.GetInternal()) {
67+
checks
68+
.ShardsLimit(1); // impl-table
69+
}
6570

6671
if (!checks) {
6772
return {CreateReject(opId, checks.GetStatus(), checks.GetError())};
@@ -93,6 +98,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
9398
*outTx.MutableLockGuard() = tx.GetLockGuard();
9499
outTx.MutableCreateTableIndex()->CopyFrom(indexDesc);
95100
outTx.MutableCreateTableIndex()->SetState(NKikimrSchemeOp::EIndexStateWriteOnly);
101+
outTx.SetInternal(tx.GetInternal());
96102

97103
if (!indexDesc.HasType()) {
98104
outTx.MutableCreateTableIndex()->SetType(NKikimrSchemeOp::EIndexTypeGlobal);
@@ -104,6 +110,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
104110
{
105111
auto outTx = TransactionTemplate(table.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexMainTable);
106112
*outTx.MutableLockGuard() = tx.GetLockGuard();
113+
outTx.SetInternal(tx.GetInternal());
107114

108115
auto& snapshot = *outTx.MutableInitiateBuildIndexMainTable();
109116
snapshot.SetTableName(table.LeafName());
@@ -113,6 +120,8 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
113120

114121
{
115122
auto outTx = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpInitiateBuildIndexImplTable);
123+
outTx.SetInternal(tx.GetInternal());
124+
116125
auto& indexImplTableDescription = *outTx.MutableCreateTable();
117126

118127
// This description provided by user to override partition policy

‎ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,13 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
8686
{
8787
auto checks = baseTablePath.Check();
8888
checks
89-
.PathShardsLimit(baseShards)
90-
.PathsLimit(pathToCreate)
91-
.ShardsLimit(shardsToCreate);
89+
.PathsLimit(pathToCreate);
90+
91+
if (!tx.GetInternal()) {
92+
checks
93+
.PathShardsLimit(baseShards)
94+
.ShardsLimit(shardsToCreate);
95+
}
9296

9397
if (!checks) {
9498
return {CreateReject(nextId, NKikimrScheme::EStatus::StatusResourceExhausted, checks.GetError())};
@@ -207,6 +211,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
207211
auto scheme = TransactionTemplate(tx.GetWorkingDir(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable);
208212
scheme.SetFailOnExist(tx.GetFailOnExist());
209213
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
214+
scheme.SetInternal(tx.GetInternal());
210215

211216
scheme.MutableCreateTable()->CopyFrom(baseTableDescription);
212217
if (tx.HasAlterUserAttributes()) {
@@ -244,6 +249,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
244249
NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable);
245250
scheme.SetFailOnExist(tx.GetFailOnExist());
246251
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
252+
scheme.SetInternal(tx.GetInternal());
247253

248254
const auto& implTableColumns = indexes.at(indexDescription.GetName());
249255

@@ -262,6 +268,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
262268
NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence);
263269
scheme.SetFailOnExist(tx.GetFailOnExist());
264270
scheme.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
271+
scheme.SetInternal(tx.GetInternal());
265272

266273
*scheme.MutableSequence() = sequenceDescription;
267274

‎ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -489,9 +489,13 @@ class TCreateTable: public TSubOperation {
489489
.IsValidLeafName()
490490
.PathsLimit()
491491
.DirChildrenLimit()
492-
.ShardsLimit(shardsToCreate)
493-
.PathShardsLimit(shardsToCreate)
494492
.IsValidACL(acl);
493+
494+
if (!Transaction.GetInternal()) {
495+
checks
496+
.ShardsLimit(shardsToCreate)
497+
.PathShardsLimit(shardsToCreate);
498+
}
495499
}
496500

497501
if (!checks) {

‎ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,12 @@ class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder
113113
checks
114114
.IsValidLeafName()
115115
.PathsLimit(2) // index and impl-table
116-
.DirChildrenLimit()
117-
.ShardsLimit(1); // impl-table
116+
.DirChildrenLimit();
117+
118+
if (!request.GetInternal()) {
119+
checks
120+
.ShardsLimit(1); // impl-table
121+
}
118122

119123
if (!checks) {
120124
return Reply(checks.GetStatus(), checks.GetError());

‎ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,9 @@ THolder<TEvIndexBuilder::TEvCreateRequest> BuildIndexPropose(
228228

229229
const TPath domainPath = TPath::Init(importInfo->DomainPathId, ss);
230230
auto propose = MakeHolder<TEvIndexBuilder::TEvCreateRequest>(ui64(txId), domainPath.PathString(), std::move(settings));
231-
(*propose->Record.MutableOperationParams()->mutable_labels())["uid"] = uid;
231+
auto& request = propose->Record;
232+
(*request.MutableOperationParams()->mutable_labels())["uid"] = uid;
233+
request.SetInternal(true);
232234

233235
return propose;
234236
}

‎ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1188,7 +1188,7 @@ value {
11881188
const auto secondTablet = TTestTxConfig::FakeHiveTablets + 1;
11891189
UpdateRow(runtime, "Original", 1, "valueA", firstTablet);
11901190
UpdateRow(runtime, "Original", 2, "valueB", secondTablet);
1191-
1191+
11921192
// Add delay after copying tables
11931193
ui64 copyTablesTxId;
11941194
auto prevObserver = runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& ev) {
@@ -4227,6 +4227,116 @@ Y_UNIT_TEST_SUITE(TImportTests) {
42274227
)", TTestTxConfig::FakeHiveTablets));
42284228
env.TestWaitNotification(runtime, txId);
42294229
}
4230+
4231+
Y_UNIT_TEST(IgnoreBasicSchemeLimits) {
4232+
TTestBasicRuntime runtime;
4233+
TTestEnv env(runtime);
4234+
ui64 txId = 100;
4235+
4236+
TestCreateExtSubDomain(runtime, ++txId, "/MyRoot", R"(
4237+
Name: "Alice"
4238+
)");
4239+
TestAlterExtSubDomain(runtime, ++txId, "/MyRoot", R"(
4240+
Name: "Alice"
4241+
ExternalSchemeShard: true
4242+
PlanResolution: 50
4243+
Coordinators: 1
4244+
Mediators: 1
4245+
TimeCastBucketsPerMediator: 2
4246+
StoragePools {
4247+
Name: "Alice:hdd"
4248+
Kind: "hdd"
4249+
}
4250+
)");
4251+
env.TestWaitNotification(runtime, txId);
4252+
4253+
ui64 tenantSchemeShard = 0;
4254+
TestDescribeResult(DescribePath(runtime, "/MyRoot/Alice"), {
4255+
NLs::ExtractTenantSchemeshard(&tenantSchemeShard)
4256+
});
4257+
UNIT_ASSERT_UNEQUAL(tenantSchemeShard, 0);
4258+
4259+
TSchemeLimits basicLimits;
4260+
basicLimits.MaxShards = 4;
4261+
basicLimits.MaxShardsInPath = 2;
4262+
SetSchemeshardSchemaLimits(runtime, basicLimits, tenantSchemeShard);
4263+
4264+
TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/Alice"), {
4265+
NLs::DomainLimitsIs(basicLimits.MaxPaths, basicLimits.MaxShards),
4266+
NLs::PathsInsideDomain(0),
4267+
NLs::ShardsInsideDomain(3)
4268+
});
4269+
4270+
TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/Alice", R"(
4271+
Name: "table1"
4272+
Columns { Name: "Key" Type: "Uint64" }
4273+
Columns { Name: "Value" Type: "Utf8" }
4274+
KeyColumnNames: ["Key"]
4275+
)");
4276+
env.TestWaitNotification(runtime, txId, tenantSchemeShard);
4277+
4278+
TestCreateTable(runtime, tenantSchemeShard, ++txId, "/MyRoot/Alice", R"(
4279+
Name: "table2"
4280+
Columns { Name: "Key" Type: "Uint64" }
4281+
Columns { Name: "Value" Type: "Utf8" }
4282+
KeyColumnNames: ["Key"]
4283+
)",
4284+
{ NKikimrScheme::StatusResourceExhausted }
4285+
);
4286+
4287+
const auto data = GenerateTestData(R"(
4288+
columns {
4289+
name: "key"
4290+
type { optional_type { item { type_id: UTF8 } } }
4291+
}
4292+
columns {
4293+
name: "value"
4294+
type { optional_type { item { type_id: UTF8 } } }
4295+
}
4296+
primary_key: "key"
4297+
partition_at_keys {
4298+
split_points {
4299+
type { tuple_type { elements { optional_type { item { type_id: UTF8 } } } } }
4300+
value { items { text_value: "b" } }
4301+
}
4302+
}
4303+
indexes {
4304+
name: "ByValue"
4305+
index_columns: "value"
4306+
global_index {}
4307+
}
4308+
)",
4309+
{{"a", 1}, {"b", 1}}
4310+
);
4311+
4312+
TPortManager portManager;
4313+
const ui16 port = portManager.GetPort();
4314+
4315+
TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port));
4316+
UNIT_ASSERT(s3Mock.Start());
4317+
4318+
const auto importId = ++txId;
4319+
TestImport(runtime, tenantSchemeShard, importId, "/MyRoot/Alice", Sprintf(R"(
4320+
ImportFromS3Settings {
4321+
endpoint: "localhost:%d"
4322+
scheme: HTTP
4323+
items {
4324+
source_prefix: ""
4325+
destination_path: "/MyRoot/Alice/ImportDir/Table"
4326+
}
4327+
}
4328+
)",
4329+
port
4330+
));
4331+
env.TestWaitNotification(runtime, importId, tenantSchemeShard);
4332+
TestGetImport(runtime, tenantSchemeShard, importId, "/MyRoot/Alice");
4333+
4334+
TestDescribeResult(DescribePath(runtime, tenantSchemeShard, "/MyRoot/Alice"), {
4335+
NLs::DomainLimitsIs(basicLimits.MaxPaths, basicLimits.MaxShards),
4336+
NLs::PathsInsideDomain(5),
4337+
NLs::ShardsInsideDomain(7)
4338+
});
4339+
}
42304340
}
42314341

42324342
Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {

0 commit comments

Comments
 (0)