Skip to content

Commit a2023e5

Browse files
Alek5andr-KotovAlek5andr-Kotov
authored
LongTxService subscription id (#6250)
1 parent 75959d3 commit a2023e5

23 files changed

+256
-80
lines changed

‎ydb/core/kqp/executer_actor/kqp_data_executer.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2594,7 +2594,9 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
25942594
auto ev = std::make_unique<TEvPersQueue::TEvProposeTransaction>();
25952595

25962596
if (writeId.Defined()) {
2597-
transaction.SetWriteId(*writeId);
2597+
auto* w = transaction.MutableWriteId();
2598+
w->SetNodeId(SelfId().NodeId());
2599+
w->SetKeyId(*writeId);
25982600
}
25992601
transaction.SetImmediate(ImmediateTx);
26002602

‎ydb/core/kqp/session_actor/kqp_session_actor.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1723,7 +1723,10 @@ class TKqpSessionActor : public TActorBootstrapped<TKqpSessionActor> {
17231723

17241724
if (replyTopicOperations) {
17251725
if (HasTopicWriteId()) {
1726-
response->MutableTopicOperations()->SetWriteId(GetTopicWriteId());
1726+
auto* w = response->MutableTopicOperations();
1727+
auto* writeId = w->MutableWriteId();
1728+
writeId->SetNodeId(SelfId().NodeId());
1729+
writeId->SetKeyId(GetTopicWriteId());
17271730
}
17281731
}
17291732

‎ydb/core/persqueue/events/internal.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <ydb/core/persqueue/key.h>
1010
#include <ydb/core/persqueue/sourceid_info.h>
1111
#include <ydb/core/persqueue/metering_sink.h>
12+
#include <ydb/core/persqueue/write_id.h>
1213
#include <ydb/core/tablet/tablet_counters.h>
1314
#include <ydb/library/persqueue/topic_parser/topic_parser.h>
1415

@@ -1158,12 +1159,12 @@ struct TEvPQ {
11581159
};
11591160

11601161
struct TEvTransactionCompleted : TEventLocal<TEvTransactionCompleted, EvTransactionCompleted> {
1161-
explicit TEvTransactionCompleted(TMaybe<ui64> writeId) :
1162+
explicit TEvTransactionCompleted(const TMaybe<NPQ::TWriteId>& writeId) :
11621163
WriteId(writeId)
11631164
{
11641165
}
11651166

1166-
TMaybe<ui64> WriteId;
1167+
TMaybe<NPQ::TWriteId> WriteId;
11671168
};
11681169
};
11691170

‎ydb/core/persqueue/partition.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3366,9 +3366,9 @@ void TPartition::ScheduleTransactionCompleted(const NKikimrPQ::TEvProposeTransac
33663366
Y_ABORT_UNLESS(tx.GetTxBodyCase() == NKikimrPQ::TEvProposeTransaction::kData);
33673367
Y_ABORT_UNLESS(tx.HasData());
33683368

3369-
TMaybe<ui64> writeId;
3369+
TMaybe<TWriteId> writeId;
33703370
if (tx.GetData().HasWriteId()) {
3371-
writeId = tx.GetData().GetWriteId();
3371+
writeId = GetWriteId(tx.GetData());
33723372
}
33733373

33743374
Replies.emplace_back(Tablet,

‎ydb/core/persqueue/partition_id.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#pragma once
22

3+
#include <ydb/core/persqueue/write_id.h>
4+
35
#include <util/generic/maybe.h>
46
#include <util/stream/output.h>
57
#include <util/system/types.h>
@@ -20,7 +22,7 @@ class TPartitionId {
2022
{
2123
}
2224

23-
TPartitionId(ui32 originalPartitionId, TMaybe<ui64> writeId, ui32 internalPartitionId) :
25+
TPartitionId(ui32 originalPartitionId, const TMaybe<TWriteId>& writeId, ui32 internalPartitionId) :
2426
OriginalPartitionId(originalPartitionId),
2527
WriteId(writeId),
2628
InternalPartitionId(internalPartitionId)
@@ -55,7 +57,7 @@ class TPartitionId {
5557
}
5658

5759
ui32 OriginalPartitionId = 0;
58-
TMaybe<ui64> WriteId;
60+
TMaybe<TWriteId> WriteId;
5961
ui32 InternalPartitionId = 0;
6062
};
6163

‎ydb/core/persqueue/pq_impl.cpp

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -957,7 +957,7 @@ void TPersQueue::InitTxWrites(const NKikimrPQ::TTabletTxInfo& info,
957957

958958
for (size_t i = 0; i != info.TxWritesSize(); ++i) {
959959
auto& txWrite = info.GetTxWrites(i);
960-
ui64 writeId = txWrite.GetWriteId();
960+
const TWriteId writeId = GetWriteId(txWrite);
961961
ui32 partitionId = txWrite.GetOriginalPartitionId();
962962
TPartitionId shadowPartitionId(partitionId, writeId, txWrite.GetInternalPartitionId());
963963

@@ -2539,7 +2539,7 @@ const TPartitionInfo& TPersQueue::GetPartitionInfo(const NKikimrClient::TPersQue
25392539
{
25402540
Y_ABORT_UNLESS(req.HasWriteId());
25412541

2542-
ui64 writeId = req.GetWriteId();
2542+
const TWriteId writeId = GetWriteId(req);
25432543
ui32 originalPartitionId = req.GetPartition();
25442544

25452545
Y_ABORT_UNLESS(TxWrites.contains(writeId) && TxWrites.at(writeId).Partitions.contains(originalPartitionId));
@@ -2624,7 +2624,7 @@ void TPersQueue::HandleEventForSupportivePartition(const ui64 responseCookie,
26242624
return;
26252625
}
26262626

2627-
ui64 writeId = req.GetWriteId();
2627+
const TWriteId writeId = GetWriteId(req);
26282628
ui32 originalPartitionId = req.GetPartition();
26292629

26302630
if (TxWrites.contains(writeId) && TxWrites.at(writeId).Partitions.contains(originalPartitionId)) {
@@ -3136,7 +3136,7 @@ void TPersQueue::Handle(TEvPersQueue::TEvProposeTransaction::TPtr& ev, const TAc
31363136
}
31373137

31383138
bool TPersQueue::CheckTxWriteOperation(const NKikimrPQ::TPartitionOperation& operation,
3139-
ui64 writeId) const
3139+
const TWriteId& writeId) const
31403140
{
31413141
TPartitionId partitionId(operation.GetPartitionId(),
31423142
writeId,
@@ -3151,7 +3151,7 @@ bool TPersQueue::CheckTxWriteOperations(const NKikimrPQ::TDataTransaction& txBod
31513151
return true;
31523152
}
31533153

3154-
ui64 writeId = txBody.GetWriteId();
3154+
const TWriteId writeId = GetWriteId(txBody);
31553155
PQ_LOG_D("writeId=" << writeId);
31563156

31573157
for (auto& operation : txBody.GetOperations()) {
@@ -3421,18 +3421,18 @@ bool TPersQueue::CanProcessTxWrites() const
34213421
return !NewSupportivePartitions.empty();
34223422
}
34233423

3424-
void TPersQueue::SubscribeWriteId(ui64 writeId,
3424+
void TPersQueue::SubscribeWriteId(const TWriteId& writeId,
34253425
const TActorContext& ctx)
34263426
{
3427-
ctx.Send(NLongTxService::MakeLongTxServiceID(ctx.SelfID.NodeId()),
3428-
new NLongTxService::TEvLongTxService::TEvSubscribeLock(writeId, ctx.SelfID.NodeId()));
3427+
ctx.Send(NLongTxService::MakeLongTxServiceID(writeId.NodeId),
3428+
new NLongTxService::TEvLongTxService::TEvSubscribeLock(writeId.KeyId, writeId.NodeId));
34293429
}
34303430

3431-
void TPersQueue::UnsubscribeWriteId(ui64 writeId,
3431+
void TPersQueue::UnsubscribeWriteId(const TWriteId& writeId,
34323432
const TActorContext& ctx)
34333433
{
3434-
ctx.Send(NLongTxService::MakeLongTxServiceID(ctx.SelfID.NodeId()),
3435-
new NLongTxService::TEvLongTxService::TEvUnsubscribeLock(writeId, ctx.SelfID.NodeId()));
3434+
ctx.Send(NLongTxService::MakeLongTxServiceID(writeId.NodeId),
3435+
new NLongTxService::TEvLongTxService::TEvUnsubscribeLock(writeId.KeyId, writeId.NodeId));
34363436
}
34373437

34383438
void TPersQueue::CreateSupportivePartitionActors(const TActorContext& ctx)
@@ -3538,7 +3538,7 @@ void TPersQueue::ProcessProposeTransactionQueue(const TActorContext& ctx)
35383538
TabletID());
35393539

35403540
if (tx.WriteId.Defined()) {
3541-
ui64 writeId = *tx.WriteId;
3541+
const TWriteId& writeId = *tx.WriteId;
35423542
Y_ABORT_UNLESS(TxWrites.contains(writeId));
35433543
TTxWriteInfo& writeInfo = TxWrites.at(writeId);
35443544
writeInfo.TxId = tx.TxId;
@@ -3717,7 +3717,7 @@ void TPersQueue::SaveTxWrites(NKikimrPQ::TTabletTxInfo& info)
37173717
for (auto& [writeId, write] : TxWrites) {
37183718
for (auto [partitionId, shadowPartitionId] : write.Partitions) {
37193719
auto* txWrite = info.MutableTxWrites()->Add();
3720-
txWrite->SetWriteId(writeId);
3720+
SetWriteId(*txWrite, writeId);
37213721
txWrite->SetOriginalPartitionId(partitionId);
37223722
txWrite->SetInternalPartitionId(shadowPartitionId.InternalPartitionId);
37233723
}
@@ -3810,7 +3810,7 @@ TMaybe<TPartitionId> TPersQueue::FindPartitionId(const NKikimrPQ::TDataTransacti
38103810
ui32 partitionId = txBody.GetOperations(0).GetPartitionId();
38113811

38123812
if (txBody.HasWriteId() && hasWriteOperation(txBody)) {
3813-
ui64 writeId = txBody.GetWriteId();
3813+
const TWriteId writeId = GetWriteId(txBody);
38143814
if (!TxWrites.contains(writeId)) {
38153815
PQ_LOG_D("unknown WriteId " << writeId);
38163816
return Nothing();
@@ -3848,7 +3848,7 @@ void TPersQueue::SendEvTxCalcPredicateToPartitions(const TActorContext& ctx,
38483848
}
38493849

38503850
if (tx.WriteId.Defined()) {
3851-
ui64 writeId = *tx.WriteId;
3851+
const TWriteId& writeId = *tx.WriteId;
38523852
Y_ABORT_UNLESS(TxWrites.contains(writeId));
38533853
const TTxWriteInfo& writeInfo = TxWrites.at(writeId);
38543854

@@ -4521,7 +4521,7 @@ void TPersQueue::ProcessCheckPartitionStatusRequests(const TPartitionId& partiti
45214521
void TPersQueue::Handle(NLongTxService::TEvLongTxService::TEvLockStatus::TPtr& ev, const TActorContext&)
45224522
{
45234523
auto& record = ev->Get()->Record;
4524-
ui64 writeId = record.GetLockId();
4524+
const TWriteId writeId(record.GetLockNode(), record.GetLockId());
45254525

45264526
if (!TxWrites.contains(writeId)) {
45274527
// the transaction has already been completed
@@ -4559,7 +4559,7 @@ void TPersQueue::Handle(TEvPQ::TEvDeletePartitionDone::TPtr& ev, const TActorCon
45594559
{
45604560
auto* event = ev->Get();
45614561
Y_ABORT_UNLESS(event->PartitionId.WriteId.Defined());
4562-
const ui64 writeId = *event->PartitionId.WriteId;
4562+
const TWriteId& writeId = *event->PartitionId.WriteId;
45634563
Y_ABORT_UNLESS(TxWrites.contains(writeId));
45644564
TTxWriteInfo& writeInfo = TxWrites.at(writeId);
45654565
Y_ABORT_UNLESS(writeInfo.Partitions.contains(event->PartitionId.OriginalPartitionId));
@@ -4593,7 +4593,7 @@ void TPersQueue::Handle(TEvPQ::TEvTransactionCompleted::TPtr& ev, const TActorCo
45934593
return;
45944594
}
45954595

4596-
const ui64 writeId = *event->WriteId;
4596+
const TWriteId& writeId = *event->WriteId;
45974597
Y_ABORT_UNLESS(TxWrites.contains(writeId));
45984598
TTxWriteInfo& writeInfo = TxWrites.at(writeId);
45994599
Y_ABORT_UNLESS(writeInfo.Partitions.size() == 1);
@@ -4604,7 +4604,7 @@ void TPersQueue::Handle(TEvPQ::TEvTransactionCompleted::TPtr& ev, const TActorCo
46044604
void TPersQueue::BeginDeleteTx(const TDistributedTransaction& tx)
46054605
{
46064606
Y_ABORT_UNLESS(tx.WriteId.Defined());
4607-
const ui64 writeId = *tx.WriteId;
4607+
const TWriteId& writeId = *tx.WriteId;
46084608
if (!TxWrites.contains(writeId)) {
46094609
// the transaction has already been completed
46104610
return;

‎ydb/core/persqueue/pq_impl.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ class TPersQueue : public NKeyValue::TKeyValueFlat {
204204
bool Deleting = false;
205205
};
206206

207-
THashMap<ui64, TTxWriteInfo> TxWrites;
207+
THashMap<TWriteId, TTxWriteInfo> TxWrites;
208208
bool TxWritesChanged = false;
209209
ui32 NextSupportivePartitionId = 100'000;
210210

@@ -488,8 +488,8 @@ class TPersQueue : public NKeyValue::TKeyValueFlat {
488488
void CreateSupportivePartitionActors(const TActorContext& ctx);
489489
void CreateSupportivePartitionActor(const TPartitionId& shadowPartitionId, const TActorContext& ctx);
490490
NKikimrPQ::TPQTabletConfig MakeSupportivePartitionConfig() const;
491-
void SubscribeWriteId(ui64 writeId, const TActorContext& ctx);
492-
void UnsubscribeWriteId(ui64 writeId, const TActorContext& ctx);
491+
void SubscribeWriteId(const TWriteId& writeId, const TActorContext& ctx);
492+
void UnsubscribeWriteId(const TWriteId& writeId, const TActorContext& ctx);
493493

494494
bool AllOriginalPartitionsInited() const;
495495

@@ -501,7 +501,7 @@ class TPersQueue : public NKeyValue::TKeyValueFlat {
501501
void BeginDeletePartitions(TTxWriteInfo& writeInfo);
502502

503503
bool CheckTxWriteOperation(const NKikimrPQ::TPartitionOperation& operation,
504-
ui64 writeId) const;
504+
const TWriteId& writeId) const;
505505
bool CheckTxWriteOperations(const NKikimrPQ::TDataTransaction& txBody) const;
506506
};
507507

‎ydb/core/persqueue/transaction.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ TDistributedTransaction::TDistributedTransaction(const NKikimrPQ::TTransaction&
4747
SourceActor = ActorIdFromProto(tx.GetSourceActor());
4848

4949
if (tx.HasWriteId()) {
50-
WriteId = tx.GetWriteId();
50+
WriteId = GetWriteId(tx);
5151
}
5252
}
5353

@@ -139,7 +139,7 @@ void TDistributedTransaction::OnProposeTransaction(const NKikimrPQ::TDataTransac
139139
InitPartitions(txBody.GetOperations());
140140

141141
if (txBody.HasWriteId() && HasWriteOperations) {
142-
WriteId = txBody.GetWriteId();
142+
WriteId = GetWriteId(txBody);
143143
} else {
144144
WriteId = Nothing();
145145
}
@@ -350,7 +350,7 @@ void TDistributedTransaction::AddCmdWriteDataTx(NKikimrPQ::TTransaction& tx)
350350
tx.SetAggrPredicate(ParticipantsDecision == NKikimrTx::TReadSetData::DECISION_COMMIT);
351351
}
352352
if (WriteId.Defined()) {
353-
tx.SetWriteId(*WriteId);
353+
SetWriteId(tx, *WriteId);
354354
}
355355
}
356356

‎ydb/core/persqueue/transaction.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ struct TDistributedTransaction {
4949
THashSet<ui64> Senders; // список отправителей TEvReadSet
5050
THashSet<ui64> Receivers; // список получателей TEvReadSet
5151
TVector<NKikimrPQ::TPartitionOperation> Operations;
52-
TMaybe<ui64> WriteId;
52+
TMaybe<TWriteId> WriteId;
5353

5454
EDecision SelfDecision = NKikimrTx::TReadSetData::DECISION_UNKNOWN;
5555
EDecision ParticipantsDecision = NKikimrTx::TReadSetData::DECISION_UNKNOWN;

‎ydb/core/persqueue/ut/internals_ut.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ Y_UNIT_TEST(StoreKeys) {
300300
TKey keyOld(TKeyPrefix::TypeData, TPartitionId{9}, 8, 7, 6, 5, false);
301301
UNIT_ASSERT_VALUES_EQUAL(keyOld.ToString(), "d0000000009_00000000000000000008_00007_0000000006_00005");
302302

303-
TKey keyNew(TKeyPrefix::TypeData, TPartitionId{5, 1, 9}, 8, 7, 6, 5, false);
303+
TKey keyNew(TKeyPrefix::TypeData, TPartitionId{5, TWriteId{0, 1}, 9}, 8, 7, 6, 5, false);
304304
UNIT_ASSERT_VALUES_EQUAL(keyNew.ToString(), "D0000000009_00000000000000000008_00007_0000000006_00005");
305305

306306
keyNew.SetType(TKeyPrefix::TypeInfo);

‎ydb/core/persqueue/ut/partition_ut.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1028,7 +1028,7 @@ void CompareVectors(const TVector<ui64>& expected, const TIterable& actual) {
10281028
}
10291029

10301030
void TPartitionFixture::ShadowPartitionCountersTest(bool isFirstClass) {
1031-
const TPartitionId partition{0, 1111, 123};
1031+
const TPartitionId partition{0, TWriteId{0, 1111}, 123};
10321032
const ui64 begin = 0;
10331033
const ui64 end = 10;
10341034
const TString session = "session";
@@ -2320,7 +2320,7 @@ Y_UNIT_TEST_F(GetPartitionWriteInfoSuccess, TPartitionFixture) {
23202320
Ctx->Runtime->GetAppData().PQConfig.MutableQuotingConfig()->SetEnableQuoting(false);
23212321

23222322
CreatePartition({
2323-
.Partition=TPartitionId{2, 10, 100'001},
2323+
.Partition=TPartitionId{2, TWriteId{0, 10}, 100'001},
23242324
//
23252325
// partition configuration
23262326
//
@@ -2389,7 +2389,7 @@ Y_UNIT_TEST_F(GetPartitionWriteInfoSuccess, TPartitionFixture) {
23892389

23902390
Y_UNIT_TEST_F(GetPartitionWriteInfoError, TPartitionFixture) {
23912391
CreatePartition({
2392-
.Partition=TPartitionId{2, 10, 100'001},
2392+
.Partition=TPartitionId{2, TWriteId{0, 10}, 100'001},
23932393
.Begin=0, .End=10,
23942394
//
23952395
// partition configuration
@@ -2444,7 +2444,7 @@ Y_UNIT_TEST_F(ShadowPartitionCountersFirstClass, TPartitionFixture) {
24442444
}
24452445

24462446
Y_UNIT_TEST_F(ShadowPartitionCountersRestore, TPartitionFixture) {
2447-
const TPartitionId partitionId{0, 1111, 123};
2447+
const TPartitionId partitionId{0, TWriteId{0, 1111}, 123};
24482448
const ui64 begin = 0;
24492449
const ui64 end = 10;
24502450
const TString session = "session";
@@ -3126,7 +3126,7 @@ Y_UNIT_TEST_F(TestBatchingWithProposeConfig, TPartitionTxTestHelper) {
31263126

31273127
Y_UNIT_TEST_F(GetUsedStorage, TPartitionFixture) {
31283128
auto* actor = CreatePartition({
3129-
.Partition=TPartitionId{2, 10, 100'001},
3129+
.Partition=TPartitionId{2, TWriteId{0, 10}, 100'001},
31303130
.Begin=0, .End=10,
31313131
//
31323132
// partition configuration

0 commit comments

Comments
 (0)