Skip to content

Commit cb022d0

Browse files
authored
YQ-2862 Added more logs to dq_pq_read_actor (#1979)
1 parent 6bd2a85 commit cb022d0

File tree

1 file changed

+26
-4
lines changed

1 file changed

+26
-4
lines changed

ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include <util/generic/algorithm.h>
3131
#include <util/generic/hash.h>
3232
#include <util/generic/utility.h>
33+
#include <util/string/join.h>
3334

3435
#include <queue>
3536
#include <variant>
@@ -91,6 +92,7 @@ struct TEvPrivate {
9192
class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqComputeActorAsyncInput {
9293
public:
9394
using TPartitionKey = std::pair<TString, ui64>; // Cluster, partition id.
95+
using TDebugOffsets = TMaybe<std::pair<ui64, ui64>>;
9496

9597
TDqPqReadActor(
9698
ui64 inputIndex,
@@ -170,8 +172,9 @@ class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqCompute
170172
data->SetVersion(StateVersion);
171173
data->SetBlob(stateBlob);
172174

173-
DeferredCommits.emplace(checkpoint.GetId(), std::move(CurrentDeferredCommit));
175+
DeferredCommits.emplace(checkpoint.GetId(), std::make_pair(std::move(CurrentDeferredCommit), CurrentDeferredCommitOffset));
174176
CurrentDeferredCommit = NYdb::NPersQueue::TDeferredCommit();
177+
CurrentDeferredCommitOffset.Clear();
175178
}
176179

177180
void LoadState(const NDqProto::TSourceState& state) override {
@@ -198,6 +201,9 @@ class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqCompute
198201
ythrow yexception() << "Invalid state version " << data.GetVersion();
199202
}
200203
}
204+
for (const auto& [key, value] : PartitionToOffset) {
205+
SRC_LOG_D("Restoring offset: cluster " << key.first << ", partition id " << key.second << ", offset: " << value);
206+
}
201207
StartingMessageTimestamp = minStartingMessageTs;
202208
IngressStats.Bytes += ingressBytes;
203209
IngressStats.Chunks++;
@@ -212,7 +218,14 @@ class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqCompute
212218
void CommitState(const NDqProto::TCheckpoint& checkpoint) override {
213219
const auto checkpointId = checkpoint.GetId();
214220
while (!DeferredCommits.empty() && DeferredCommits.front().first <= checkpointId) {
215-
DeferredCommits.front().second.Commit();
221+
auto& valuePair = DeferredCommits.front().second;
222+
const auto& offsets = valuePair.second;
223+
if (offsets.Empty()) {
224+
SRC_LOG_D("Commit offset: [ empty ]");
225+
} else {
226+
SRC_LOG_D("Commit offset: [" << offsets->first << ", " << offsets->second << "]");
227+
}
228+
valuePair.first.Commit();
216229
DeferredCommits.pop();
217230
}
218231
}
@@ -364,7 +377,9 @@ class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqCompute
364377
NYdb::NPersQueue::TReadSessionSettings GetReadSessionSettings() const {
365378
NYdb::NPersQueue::TTopicReadSettings topicReadSettings;
366379
topicReadSettings.Path(SourceParams.GetTopicPath());
367-
for (const auto partitionId : GetPartitionsToRead()) {
380+
auto partitionsToRead = GetPartitionsToRead();
381+
SRC_LOG_D("PartitionsToRead: " << JoinSeq(", ", partitionsToRead));
382+
for (const auto partitionId : partitionsToRead) {
368383
topicReadSettings.AppendPartitionGroupIds(partitionId);
369384
}
370385

@@ -419,6 +434,12 @@ class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqCompute
419434
for (const auto& [partitionStream, ranges] : readyBatch.OffsetRanges) {
420435
for (const auto& [start, end] : ranges) {
421436
CurrentDeferredCommit.Add(partitionStream, start, end);
437+
if (!CurrentDeferredCommitOffset) {
438+
CurrentDeferredCommitOffset = std::make_pair(start, end);
439+
} else {
440+
CurrentDeferredCommitOffset->first = std::min(CurrentDeferredCommitOffset->first, start);
441+
CurrentDeferredCommitOffset->second = std::max(CurrentDeferredCommitOffset->second, end);
442+
}
422443
}
423444
PartitionToOffset[MakePartitionKey(partitionStream)] = ranges.back().second;
424445
}
@@ -574,8 +595,9 @@ class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqCompute
574595
THashMap<TPartitionKey, ui64> PartitionToOffset; // {cluster, partition} -> offset of next event.
575596
TInstant StartingMessageTimestamp;
576597
const NActors::TActorId ComputeActorId;
577-
std::queue<std::pair<ui64, NYdb::NPersQueue::TDeferredCommit>> DeferredCommits;
598+
std::queue<std::pair<ui64, std::pair<NYdb::NPersQueue::TDeferredCommit, TDebugOffsets>>> DeferredCommits;
578599
NYdb::NPersQueue::TDeferredCommit CurrentDeferredCommit;
600+
TDebugOffsets CurrentDeferredCommitOffset;
579601
bool SubscribedOnEvent = false;
580602
std::vector<std::tuple<TString, TPqMetaExtractor::TPqMetaExtractorLambda>> MetadataFields;
581603
std::queue<TReadyBatch> ReadyBuffer;

0 commit comments

Comments
 (0)