Skip to content

Commit 4f41262

Browse files
vitalifvitalif
committed
Add StartTime/EndTime/UserSID for BuildIndex, fix listing order (newest first instead of random) (ydb-platform#16971)
1 parent 16ba646 commit 4f41262

16 files changed

+87
-30
lines changed

‎ydb/core/grpc_services/operation_helpers.cpp

+9
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,15 @@ Ydb::TOperationId ToOperationId(const NKikimrIndexBuilder::TIndexBuild& build) {
5959
void ToOperation(const NKikimrIndexBuilder::TIndexBuild& build, Ydb::Operations::Operation* operation) {
6060
operation->set_id(NOperationId::ProtoToString(ToOperationId(build)));
6161
operation->mutable_issues()->CopyFrom(build.GetIssues());
62+
if (build.HasStartTime()) {
63+
*operation->mutable_create_time() = build.GetStartTime();
64+
}
65+
if (build.HasEndTime()) {
66+
*operation->mutable_end_time() = build.GetEndTime();
67+
}
68+
if (build.HasUserSID()) {
69+
operation->set_created_by(build.GetUserSID());
70+
}
6271

6372
switch (build.GetState()) {
6473
case Ydb::Table::IndexBuildState::STATE_DONE:

‎ydb/core/grpc_services/rpc_alter_table.cpp

+3
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,9 @@ class TAlterTableRPC : public TRpcSchemeRequestActor<TAlterTableRPC, TEvAlterTab
342342
void SendAddIndexOpToSS(const TActorContext& ctx, ui64 schemeShardId) {
343343
SetSchemeShardId(schemeShardId);
344344
auto ev = std::make_unique<NSchemeShard::TEvIndexBuilder::TEvCreateRequest>(TxId, DatabaseName, std::move(IndexBuildSettings));
345+
if (UserToken) {
346+
ev->Record.SetUserSID(UserToken->GetUserSID());
347+
}
345348
ForwardToSchemeShard(ctx, std::move(ev));
346349
}
347350

‎ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp

+3
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,9 @@ class TKqpSchemeExecuter : public TActorBootstrapped<TKqpSchemeExecuter> {
697697
const auto& buildOp = schemeOp.GetBuildOperation();
698698
SetSchemeShardId(domainInfo->ExtractSchemeShard());
699699
auto req = std::make_unique<NSchemeShard::TEvIndexBuilder::TEvCreateRequest>(TxId, Database, buildOp);
700+
if (UserToken) {
701+
req->Record.SetUserSID(UserToken->GetUserSID());
702+
}
700703
ForwardToSchemeShard(std::move(req));
701704
}
702705

‎ydb/core/protos/index_builder.proto

+5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import "ydb/public/api/protos/ydb_status_codes.proto";
44
import "ydb/public/api/protos/ydb_table.proto";
55
import "ydb/public/api/protos/ydb_value.proto";
66
import "google/protobuf/any.proto";
7+
import "google/protobuf/timestamp.proto";
78

89

910
package NKikimrIndexBuilder;
@@ -51,6 +52,9 @@ message TIndexBuild {
5152
optional Ydb.Table.IndexBuildState.State State = 3;
5253
optional TIndexBuildSettings Settings = 4;
5354
optional float Progress = 5 [default = 0];
55+
optional google.protobuf.Timestamp StartTime = 6;
56+
optional google.protobuf.Timestamp EndTime = 7;
57+
optional string UserSID = 8;
5458
}
5559

5660
message TEvCreateRequest {
@@ -60,6 +64,7 @@ message TEvCreateRequest {
6064
optional TIndexBuildSettings Settings = 4;
6165
// Internal flag is true for system-generated operations and is false for those initiated directly by the user.
6266
optional bool Internal = 5 [default = false];
67+
optional string UserSID = 6;
6368
}
6469

6570
message TEvCreateResponse {

‎ydb/core/tx/schemeshard/schemeshard__init.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -3410,7 +3410,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
34103410
}
34113411
}
34123412

3413-
// Read KesusAlters
3413+
// Read KesusAlters
34143414
{
34153415
TKesusAlterRows kesusAlterRows;
34163416
if (!LoadKesusAlters(db, kesusAlterRows)) {

‎ydb/core/tx/schemeshard/schemeshard_build_index.cpp

+12-2
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,14 @@ void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuil
6767
NIceDb::TUpdate<Schema::IndexBuild::MaxBatchBytes>(info.ScanSettings.GetMaxBatchBytes()),
6868
NIceDb::TUpdate<Schema::IndexBuild::MaxShards>(info.MaxInProgressShards),
6969
NIceDb::TUpdate<Schema::IndexBuild::MaxRetries>(info.ScanSettings.GetMaxBatchRetries()),
70-
NIceDb::TUpdate<Schema::IndexBuild::BuildKind>(ui32(info.BuildKind))
70+
NIceDb::TUpdate<Schema::IndexBuild::BuildKind>(ui32(info.BuildKind)),
71+
NIceDb::TUpdate<Schema::IndexBuild::StartTime>(info.StartTime.Seconds())
7172
);
73+
if (info.UserSID) {
74+
persistedBuildIndex.Update(
75+
NIceDb::TUpdate<Schema::IndexBuild::UserSID>(*info.UserSID)
76+
);
77+
}
7278
// Persist details of the index build operation: ImplTableDescriptions and SpecializedIndexDescription.
7379
// We have chosen TIndexCreationConfig's string representation as the serialization format.
7480
if (bool hasSpecializedDescription = !std::holds_alternative<std::monostate>(info.SpecializedIndexDescription);
@@ -119,7 +125,11 @@ void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuil
119125

120126
void TSchemeShard::PersistBuildIndexState(NIceDb::TNiceDb& db, const TIndexBuildInfo& indexInfo) {
121127
db.Table<Schema::IndexBuild>().Key(indexInfo.Id).Update(
122-
NIceDb::TUpdate<Schema::IndexBuild::State>(ui32(indexInfo.State)));
128+
NIceDb::TUpdate<Schema::IndexBuild::State>(ui32(indexInfo.State)),
129+
NIceDb::TUpdate<Schema::IndexBuild::Issue>(indexInfo.Issue),
130+
NIceDb::TUpdate<Schema::IndexBuild::StartTime>(indexInfo.StartTime.Seconds()),
131+
NIceDb::TUpdate<Schema::IndexBuild::EndTime>(indexInfo.EndTime.Seconds())
132+
);
123133
}
124134

125135
void TSchemeShard::PersistBuildIndexCancelRequest(NIceDb::TNiceDb& db, const TIndexBuildInfo& indexInfo) {

‎ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp

+4
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,10 @@ class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder
182182

183183
buildInfo->CreateSender = Request->Sender;
184184
buildInfo->SenderCookie = Request->Cookie;
185+
buildInfo->StartTime = ctx.Now();
186+
if (request.HasUserSID()) {
187+
buildInfo->UserSID = request.GetUserSID();
188+
}
185189

186190
Self->PersistCreateBuildIndex(db, *buildInfo);
187191

‎ydb/core/tx/schemeshard/schemeshard_build_index__list.cpp

+6-6
Original file line numberDiff line numberDiff line change
@@ -43,28 +43,28 @@ struct TSchemeShard::TIndexBuilder::TTxList: public TSchemeShard::TIndexBuilder:
4343
const ui64 pageSize = Min(record.GetPageSize() ? Max(record.GetPageSize(), MinPageSize) : DefaultPageSize, MaxPageSize);
4444

4545

46-
auto it = Self->IndexBuilds.begin();
46+
auto it = Self->IndexBuilds.end();
4747
ui64 skip = (page - 1) * pageSize;
48-
while ((it != Self->IndexBuilds.end()) && skip) {
48+
while ((it != Self->IndexBuilds.begin()) && skip) {
49+
--it;
4950
if (it->second->DomainPathId == domainPathId) {
5051
--skip;
5152
}
52-
++it;
5353
}
5454

5555
auto& respRecord = Response->Record;
5656
respRecord.SetStatus(Ydb::StatusIds::SUCCESS);
5757

5858
ui64 size = 0;
59-
while ((it != Self->IndexBuilds.end()) && size < pageSize) {
59+
while ((it != Self->IndexBuilds.begin()) && size < pageSize) {
60+
--it;
6061
if (it->second->DomainPathId == domainPathId) {
6162
Fill(*respRecord.MutableEntries()->Add(), *it->second);
6263
++size;
6364
}
64-
++it;
6565
}
6666

67-
if (it == Self->IndexBuilds.end()) {
67+
if (it == Self->IndexBuilds.begin()) {
6868
respRecord.SetNextPageToken("0");
6969
} else {
7070
respRecord.SetNextPageToken(ToString(page + 1));

‎ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp

+16-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
namespace NKikimr {
1414
namespace NSchemeShard {
1515

16-
void TSchemeShard::TIndexBuilder::TTxBase::ApplyState(NTabletFlatExecutor::TTransactionContext& txc) {
16+
void TSchemeShard::TIndexBuilder::TTxBase::ApplyState(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) {
1717
for (auto& rec: StateChanges) {
1818
TIndexBuildId buildId;
1919
TIndexBuildInfo::EState state;
@@ -23,6 +23,11 @@ void TSchemeShard::TIndexBuilder::TTxBase::ApplyState(NTabletFlatExecutor::TTran
2323
Y_VERIFY_S(buildInfoPtr, "IndexBuilds has no " << buildId);
2424
auto& buildInfo = *buildInfoPtr->Get();
2525
LOG_I("Change state from " << buildInfo.State << " to " << state);
26+
if (state == TIndexBuildInfo::EState::Rejected ||
27+
state == TIndexBuildInfo::EState::Cancelled ||
28+
state == TIndexBuildInfo::EState::Done) {
29+
buildInfo.EndTime = ctx.Now();
30+
}
2631
buildInfo.State = state;
2732

2833
NIceDb::TNiceDb db(txc.DB);
@@ -32,7 +37,7 @@ void TSchemeShard::TIndexBuilder::TTxBase::ApplyState(NTabletFlatExecutor::TTran
3237

3338
void TSchemeShard::TIndexBuilder::TTxBase::ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) {
3439
SideEffects.ApplyOnExecute(Self, txc, ctx);
35-
ApplyState(txc);
40+
ApplyState(txc, ctx);
3641
ApplyBill(txc, ctx);
3742
}
3843

@@ -195,6 +200,15 @@ void TSchemeShard::TIndexBuilder::TTxBase::Fill(NKikimrIndexBuilder::TIndexBuild
195200
if (indexInfo.Issue) {
196201
AddIssue(index.MutableIssues(), indexInfo.Issue);
197202
}
203+
if (indexInfo.StartTime != TInstant::Zero()) {
204+
*index.MutableStartTime() = SecondsToProtoTimeStamp(indexInfo.StartTime.Seconds());
205+
}
206+
if (indexInfo.EndTime != TInstant::Zero()) {
207+
*index.MutableEndTime() = SecondsToProtoTimeStamp(indexInfo.EndTime.Seconds());
208+
}
209+
if (indexInfo.UserSID) {
210+
index.SetUserSID(*indexInfo.UserSID);
211+
}
198212

199213
for (const auto& item: indexInfo.Shards) {
200214
const TShardIdx& shardIdx = item.first;

‎ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class TSchemeShard::TIndexBuilder::TTxBase: public NTabletFlatExecutor::TTransac
2222
using TToBill = std::tuple<TIndexBuildId, TInstant, TInstant>;
2323
TDeque<TToBill> ToBill;
2424

25-
void ApplyState(NTabletFlatExecutor::TTransactionContext& txc);
25+
void ApplyState(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx);
2626
void ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx);
2727
void ApplyOnComplete(const TActorContext& ctx);
2828
void ApplySchedule(const TActorContext& ctx);

‎ydb/core/tx/schemeshard/schemeshard_export.cpp

-7
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,6 @@ namespace {
2929
}
3030
}
3131

32-
NProtoBuf::Timestamp SecondsToProtoTimeStamp(ui64 sec) {
33-
NProtoBuf::Timestamp timestamp;
34-
timestamp.set_seconds((i64)(sec));
35-
timestamp.set_nanos(0);
36-
return timestamp;
37-
}
38-
3932
void FillItemProgress(TSchemeShard* ss, const TExportInfo::TPtr exportInfo, ui32 itemIdx,
4033
Ydb::Export::ExportItemProgress& itemProgress) {
4134

‎ydb/core/tx/schemeshard/schemeshard_impl.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -1365,7 +1365,7 @@ class TSchemeShard
13651365
// namespace NIndexBuilder {
13661366
TControlWrapper AllowDataColumnForIndexTable;
13671367

1368-
THashMap<TIndexBuildId, TIndexBuildInfo::TPtr> IndexBuilds;
1368+
TMap<TIndexBuildId, TIndexBuildInfo::TPtr> IndexBuilds;
13691369
THashMap<TString, TIndexBuildInfo::TPtr> IndexBuildsByUid;
13701370
THashMap<TTxId, TIndexBuildId> TxIdToIndexBuilds;
13711371

‎ydb/core/tx/schemeshard/schemeshard_import.cpp

-7
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,6 @@ namespace {
2121
}
2222
}
2323

24-
NProtoBuf::Timestamp SecondsToProtoTimeStamp(ui64 sec) {
25-
NProtoBuf::Timestamp timestamp;
26-
timestamp.set_seconds((i64)(sec));
27-
timestamp.set_nanos(0);
28-
return timestamp;
29-
}
30-
3124
TImportInfo::EState GetMinState(TImportInfo::TPtr importInfo) {
3225
TImportInfo::EState state = TImportInfo::EState::Invalid;
3326

‎ydb/core/tx/schemeshard/schemeshard_info_types.cpp

+7
Original file line numberDiff line numberDiff line change
@@ -2619,5 +2619,12 @@ std::optional<std::pair<i64, i64>> ValidateSequenceType(const TString& sequenceN
26192619
return {{dataTypeMinValue, dataTypeMaxValue}};
26202620
}
26212621

2622+
NProtoBuf::Timestamp SecondsToProtoTimeStamp(ui64 sec) {
2623+
NProtoBuf::Timestamp timestamp;
2624+
timestamp.set_seconds((i64)(sec));
2625+
timestamp.set_nanos(0);
2626+
return timestamp;
2627+
}
2628+
26222629
} // namespace NSchemeShard
26232630
} // namespace NKikimr

‎ydb/core/tx/schemeshard/schemeshard_info_types.h

+11-2
Original file line numberDiff line numberDiff line change
@@ -3097,6 +3097,7 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
30973097

30983098
TIndexBuildId Id;
30993099
TString Uid;
3100+
TMaybe<TString> UserSID;
31003101

31013102
TPathId DomainPathId;
31023103
TPathId TablePathId;
@@ -3320,6 +3321,8 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
33203321

33213322
EState State = EState::Invalid;
33223323
TString Issue;
3324+
TInstant StartTime = TInstant::Zero();
3325+
TInstant EndTime = TInstant::Zero();
33233326

33243327
TSet<TActorId> Subscribers;
33253328

@@ -3593,6 +3596,11 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
35933596
row.template GetValueOrDefault<Schema::IndexBuild::Issue>();
35943597
indexInfo->CancelRequested =
35953598
row.template GetValueOrDefault<Schema::IndexBuild::CancelRequest>(false);
3599+
if (row.template HaveValue<Schema::IndexBuild::UserSID>()) {
3600+
indexInfo->UserSID = row.template GetValue<Schema::IndexBuild::UserSID>();
3601+
}
3602+
indexInfo->StartTime = TInstant::Seconds(row.template GetValueOrDefault<Schema::IndexBuild::StartTime>());
3603+
indexInfo->EndTime = TInstant::Seconds(row.template GetValueOrDefault<Schema::IndexBuild::EndTime>());
35963604

35973605
indexInfo->LockTxId =
35983606
row.template GetValueOrDefault<Schema::IndexBuild::LockTxId>(
@@ -3870,9 +3878,10 @@ TConclusion<TDuration> GetExpireAfter(const NKikimrSchemeOp::TTTLSettings::TEnab
38703878
std::optional<std::pair<i64, i64>> ValidateSequenceType(const TString& sequenceName, const TString& dataType,
38713879
const NKikimr::NScheme::TTypeRegistry& typeRegistry, bool pgTypesEnabled, TString& errStr);
38723880

3873-
}
3881+
NProtoBuf::Timestamp SecondsToProtoTimeStamp(ui64 sec);
38743882

3875-
}
3883+
} // namespace NSchemeShard
3884+
} // namespace NKikimr
38763885

38773886
template <>
38783887
inline void Out<NKikimr::NSchemeShard::TIndexBuildInfo::TShardStatus>

‎ydb/core/tx/schemeshard/schemeshard_schema.h

+8-1
Original file line numberDiff line numberDiff line change
@@ -1377,6 +1377,10 @@ struct Schema : NIceDb::Schema {
13771377
struct ReadRowsProcessed : Column<39, NScheme::NTypeIds::Uint64> {};
13781378
struct ReadBytesProcessed : Column<40, NScheme::NTypeIds::Uint64> {};
13791379

1380+
struct StartTime : Column<41, NScheme::NTypeIds::Uint64> {};
1381+
struct EndTime : Column<42, NScheme::NTypeIds::Uint64> {};
1382+
struct UserSID : Column<43, NScheme::NTypeIds::Utf8> {};
1383+
13801384
using TKey = TableKey<Id>;
13811385
using TColumns = TableColumns<
13821386
Id,
@@ -1418,7 +1422,10 @@ struct Schema : NIceDb::Schema {
14181422
UploadRowsProcessed,
14191423
UploadBytesProcessed,
14201424
ReadRowsProcessed,
1421-
ReadBytesProcessed
1425+
ReadBytesProcessed,
1426+
StartTime,
1427+
EndTime,
1428+
UserSID
14221429
>;
14231430
};
14241431

0 commit comments

Comments
 (0)