Skip to content

Commit 22af3e0

Browse files
abyss7abyss7
authored
Merge d5afc6b into 3985116 
2 parents 3985116 + d5afc6b commit 22af3e0

File tree

5 files changed

+52
-34
lines changed

5 files changed

+52
-34
lines changed

‎ydb/core/kqp/executer_actor/kqp_data_executer.cpp

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -529,7 +529,7 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
529529
if (ev->Get()->Record.GetState() == NDqProto::COMPUTE_STATE_FAILURE) {
530530
CancelProposal(0);
531531
}
532-
HandleComputeStats(ev);
532+
HandleComputeState(ev);
533533
}
534534

535535
void HandlePrepare(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) {
@@ -1009,7 +1009,7 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
10091009
hFunc(TEvInterconnect::TEvNodeDisconnected, HandleDisconnected);
10101010
hFunc(TEvKqpNode::TEvStartKqpTasksResponse, HandleStartKqpTasksResponse);
10111011
hFunc(TEvTxProxy::TEvProposeTransactionStatus, HandleExecute);
1012-
hFunc(TEvDqCompute::TEvState, HandleComputeStats);
1012+
hFunc(TEvDqCompute::TEvState, HandleComputeState);
10131013
hFunc(NYql::NDq::TEvDqCompute::TEvChannelData, HandleChannelData);
10141014
hFunc(TEvKqpExecuter::TEvStreamDataAck, HandleStreamAck);
10151015
hFunc(TEvKqp::TEvAbortExecution, HandleExecute);
@@ -2608,6 +2608,7 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
26082608
this->Become(&TThis::WaitShutdownState);
26092609
LOG_I("Waiting for shutdown of " << Planner->GetPendingComputeTasks().size() << " tasks and "
26102610
<< Planner->GetPendingComputeActors().size() << " compute actors");
2611+
// TODO(ilezhankin): the CA awaiting timeout should be configurable.
26112612
TActivationContext::Schedule(TDuration::Seconds(10), new IEventHandle(SelfId(), SelfId(), new TEvents::TEvPoison));
26122613
}
26132614
} else {
@@ -2643,17 +2644,10 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
26432644
}
26442645

26452646
void HandleShutdown(TEvDqCompute::TEvState::TPtr& ev) {
2646-
if (ev->Get()->Record.GetState() == NDqProto::COMPUTE_STATE_FAILURE) {
2647-
YQL_ENSURE(Planner);
2648-
2649-
TActorId actor = ev->Sender;
2650-
ui64 taskId = ev->Get()->Record.GetTaskId();
2651-
2652-
Planner->CompletedCA(taskId, actor);
2647+
HandleComputeStats(ev);
26532648

2654-
if (Planner->GetPendingComputeTasks().empty() && Planner->GetPendingComputeActors().empty()) {
2655-
PassAway();
2656-
}
2649+
if (Planner->GetPendingComputeTasks().empty() && Planner->GetPendingComputeActors().empty()) {
2650+
PassAway();
26572651
}
26582652
}
26592653

‎ydb/core/kqp/executer_actor/kqp_executer_impl.h

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,7 @@ class TKqpExecuterBase : public TActorBootstrapped<TDerived> {
380380
this->Send(channelComputeActorId, ackEv.Release(), /* TODO: undelivery */ 0, /* cookie */ channelId);
381381
}
382382

383-
void HandleComputeStats(NYql::NDq::TEvDqCompute::TEvState::TPtr& ev) {
383+
bool HandleComputeStats(NYql::NDq::TEvDqCompute::TEvState::TPtr& ev) {
384384
TActorId computeActor = ev->Sender;
385385
auto& state = ev->Get()->Record;
386386
ui64 taskId = state.GetTaskId();
@@ -409,7 +409,40 @@ class TKqpExecuterBase : public TActorBootstrapped<TDerived> {
409409
}
410410

411411
YQL_ENSURE(Planner);
412-
bool populateChannels = Planner->AcknowledgeCA(taskId, computeActor, &state);
412+
bool ack = Planner->AcknowledgeCA(taskId, computeActor, &state);
413+
414+
// Don't finalize stats twice.
415+
if (Planner->CompletedCA(taskId, computeActor)) {
416+
switch (state.GetState()) {
417+
case NYql::NDqProto::COMPUTE_STATE_FAILURE:
418+
case NYql::NDqProto::COMPUTE_STATE_FINISHED: {
419+
ExtraData[computeActor].Swap(state.MutableExtraData());
420+
421+
if (Stats) {
422+
Stats->AddComputeActorStats(
423+
computeActor.NodeId(),
424+
std::move(*state.MutableStats()),
425+
TDuration::MilliSeconds(AggregationSettings.GetCollectLongTasksStatsTimeoutMs())
426+
);
427+
}
428+
429+
LastTaskId = taskId;
430+
LastComputeActorId = computeActor.ToString();
431+
}
432+
default:
433+
; // ignore all other states.
434+
}
435+
}
436+
437+
return ack;
438+
}
439+
440+
void HandleComputeState(NYql::NDq::TEvDqCompute::TEvState::TPtr& ev) {
441+
TActorId computeActor = ev->Sender;
442+
auto& state = ev->Get()->Record;
443+
ui64 taskId = state.GetTaskId();
444+
445+
bool populateChannels = HandleComputeStats(ev);
413446

414447
switch (state.GetState()) {
415448
case NYql::NDqProto::COMPUTE_STATE_UNKNOWN: {
@@ -427,22 +460,8 @@ class TKqpExecuterBase : public TActorBootstrapped<TDerived> {
427460
break;
428461
}
429462

430-
case NYql::NDqProto::COMPUTE_STATE_FAILURE:
431-
case NYql::NDqProto::COMPUTE_STATE_FINISHED: {
432-
ExtraData[computeActor].Swap(state.MutableExtraData());
433-
if (Stats) {
434-
Stats->AddComputeActorStats(
435-
computeActor.NodeId(),
436-
std::move(*state.MutableStats()),
437-
TDuration::MilliSeconds(AggregationSettings.GetCollectLongTasksStatsTimeoutMs())
438-
);
439-
}
440-
441-
LastTaskId = taskId;
442-
LastComputeActorId = computeActor.ToString();
443-
YQL_ENSURE(Planner);
444-
Planner->CompletedCA(taskId, computeActor);
445-
}
463+
default:
464+
; // ignore all other states.
446465
}
447466

448467
if (state.GetState() == NYql::NDqProto::COMPUTE_STATE_FAILURE) {
@@ -1854,6 +1873,9 @@ class TKqpExecuterBase : public TActorBootstrapped<TDerived> {
18541873

18551874
void PassAway() override {
18561875
YQL_ENSURE(AlreadyReplied && ResponseEv);
1876+
1877+
// Actualize stats with the last stats from terminated CAs, but keep the status.
1878+
FillResponseStats(ResponseEv->Record.GetResponse().GetStatus());
18571879
this->Send(Target, ResponseEv.release());
18581880

18591881
for (auto channelPair: ResultChannelProxies) {

‎ydb/core/kqp/executer_actor/kqp_planner.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -551,11 +551,11 @@ bool TKqpPlanner::AcknowledgeCA(ui64 taskId, TActorId computeActor, const NYql::
551551
return false;
552552
}
553553

554-
void TKqpPlanner::CompletedCA(ui64 taskId, TActorId computeActor) {
554+
bool TKqpPlanner::CompletedCA(ui64 taskId, TActorId computeActor) {
555555
auto& task = TasksGraph.GetTask(taskId);
556556
if (task.Meta.Completed) {
557557
YQL_ENSURE(!PendingComputeActors.contains(computeActor));
558-
return;
558+
return false;
559559
}
560560

561561
task.Meta.Completed = true;
@@ -565,6 +565,8 @@ void TKqpPlanner::CompletedCA(ui64 taskId, TActorId computeActor) {
565565
PendingComputeActors.erase(it);
566566

567567
LOG_I("Compute actor has finished execution: " << computeActor.ToString());
568+
569+
return true;
568570
}
569571

570572
void TKqpPlanner::TaskNotStarted(ui64 taskId) {

‎ydb/core/kqp/executer_actor/kqp_planner.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ class TKqpPlanner {
7373
std::unique_ptr<IEventHandle> PlanExecution();
7474
std::unique_ptr<IEventHandle> AssignTasksToNodes();
7575
bool AcknowledgeCA(ui64 taskId, TActorId computeActor, const NYql::NDqProto::TEvComputeActorState* state);
76-
void CompletedCA(ui64 taskId, TActorId computeActor);
76+
bool CompletedCA(ui64 taskId, TActorId computeActor);
7777
void TaskNotStarted(ui64 taskId);
7878
TProgressStat::TEntry CalculateConsumptionUpdate();
7979
void ShiftConsumption();

‎ydb/core/kqp/executer_actor/kqp_scan_executer.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ class TKqpScanExecuter : public TKqpExecuterBase<TKqpScanExecuter, EExecType::Sc
110110
STATEFN(ExecuteState) {
111111
try {
112112
switch (ev->GetTypeRewrite()) {
113-
hFunc(TEvDqCompute::TEvState, HandleComputeStats);
113+
hFunc(TEvDqCompute::TEvState, HandleComputeState);
114114
hFunc(TEvDqCompute::TEvChannelData, HandleChannelData); // from CA
115115
hFunc(TEvKqpExecuter::TEvStreamDataAck, HandleStreamAck);
116116
hFunc(TEvKqp::TEvAbortExecution, HandleAbortExecution);

0 commit comments

Comments
 (0)