# This is a combination of 11 commits.

# This is the 1st commit message:

test utils: refactor fail_payment_along_route for mpp

# This is the commit message #2:

Add MppId field to HTLCSource as a way to correlate mpp payment paths

# This is the commit message #3:

Implement Readable/Writeable for HashSet

To be used in upcoming commits for MPP ID storage

# This is the commit message #4:

Add MPP ID to pending_outbound_htlcs

We'll use this to correlate MPP shards in upcoming commits

# This is the commit message #5:

Prevent duplicate PaymentSent events

by removing all pending outbound payments associated with the same
MPP payment after the preimage is received

# This is the commit message #6:

Add all_paths_failed field to PaymentFailed

see field docs for details

# This is the commit message #7:

Drop writer size hinting/message vec preallocation

In order to avoid significant malloc traffic, messages previously
explicitly stated their serialized length allowing for Vec
preallocation during the message serialization pipeline. This added
some amount of complexity in the serialization code, but did avoid
some realloc() calls.

Instead, here, we drop all the complexity in favor of a fixed 2KiB
buffer for all message serialization. This should not only be
simpler with a similar reduction in realloc() traffic, but also
may reduce heap fragmentation by allocating identically-sized
buffers more often.

# This is the commit message #8:

[fuzz] Swap mode on most messages to account for TLV suffix

# This is the commit message #9:

Add forward-compat due serialization variants of HTLCFailureMsg

Going forward, all lightning messages have a TLV stream suffix,
allowing new fields to be added as needed. In the P2P protocol,
messages have an explicit length, so there is no implied length in
the TLV stream itself. HTLCFailureMsg enum variants have messages
in them, but without a size prefix or any explicit end. Thus, if a
HTLCFailureMsg is read as a part of a ChannelManager, with a TLV
stream at the end, there is no way to differentiate between the end
of the message and the next field(s) in the ChannelManager.

Here we add two new variant values for HTLCFailureMsg variants in
the read path, allowing us to switch to the new values if/when we
add new TLV fields in UpdateFailHTLC or UpdateFailMalformedHTLC so
that older versions can still read the new TLV fields.

# This is the commit message #10:

Convert most P2P msg serialization to a new macro with TLV suffixes

The network serialization format for all messages was changed some
time ago to include a TLV suffix for all messages, however we never
bothered to implement it as there isn't a lot of use validating a
TLV stream with nothing to do with it. However, messages are
increasingly utilizing the TLV suffix feature, and there are some
compatibility concerns with messages written as a part of other
structs having their format changed (see previous commit).

Thus, here we go ahead and convert most message serialization to a
new macro which includes a TLV suffix after a series of fields,
simplifying several serialization implementations in the process.

# This is the commit message #11:

Update docs to specify where process events is called

Author: valentinewallace

fuzz/src/chanmon_consistency.rs

@@ -94,9 +94,6 @@ impl Writer for VecWriter {
 		self.0.extend_from_slice(buf);
 		Ok(())
 	}
-	fn size_hint(&mut self, size: usize) {
-		self.0.reserve_exact(size);
-	}
 }
 
 struct TestChainMonitor {

fuzz/src/chanmon_deser.rs

@@ -18,9 +18,6 @@ impl Writer for VecWriter {
 		self.0.extend_from_slice(buf);
 		Ok(())
 	}
-	fn size_hint(&mut self, size: usize) {
-		self.0.reserve_exact(size);
-	}
 }
 
 #[inline]

fuzz/src/msg_targets/gen_target.sh

@@ -11,36 +11,36 @@ echo "mod utils;" > mod.rs
 
 # Note when adding new targets here you should add a similar line in src/bin/gen_target.sh
 
-GEN_TEST AcceptChannel test_msg ""
-GEN_TEST AnnouncementSignatures test_msg ""
+GEN_TEST AcceptChannel test_msg_simple ""
+GEN_TEST AnnouncementSignatures test_msg_simple ""
+GEN_TEST ClosingSigned test_msg_simple ""
+GEN_TEST CommitmentSigned test_msg_simple ""
+GEN_TEST FundingCreated test_msg_simple ""
+GEN_TEST FundingLocked test_msg_simple ""
+GEN_TEST FundingSigned test_msg_simple ""
+GEN_TEST GossipTimestampFilter test_msg_simple ""
+GEN_TEST Init test_msg_simple ""
+GEN_TEST OnionHopData test_msg_simple ""
+GEN_TEST OpenChannel test_msg_simple ""
+GEN_TEST Ping test_msg_simple ""
+GEN_TEST Pong test_msg_simple ""
+GEN_TEST QueryChannelRange test_msg_simple ""
+GEN_TEST ReplyShortChannelIdsEnd test_msg_simple ""
+GEN_TEST RevokeAndACK test_msg_simple ""
+GEN_TEST Shutdown test_msg_simple ""
+GEN_TEST UpdateAddHTLC test_msg_simple ""
+GEN_TEST UpdateFailHTLC test_msg_simple ""
+GEN_TEST UpdateFailMalformedHTLC test_msg_simple ""
+GEN_TEST UpdateFee test_msg_simple ""
+GEN_TEST UpdateFulfillHTLC test_msg_simple ""
+
 GEN_TEST ChannelReestablish test_msg ""
-GEN_TEST CommitmentSigned test_msg ""
 GEN_TEST DecodedOnionErrorPacket test_msg ""
-GEN_TEST FundingCreated test_msg ""
-GEN_TEST FundingLocked test_msg ""
-GEN_TEST FundingSigned test_msg ""
-GEN_TEST OpenChannel test_msg ""
-GEN_TEST RevokeAndACK test_msg ""
-GEN_TEST Shutdown test_msg ""
-GEN_TEST UpdateFailHTLC test_msg ""
-GEN_TEST UpdateFailMalformedHTLC test_msg ""
-GEN_TEST UpdateFee test_msg ""
-GEN_TEST UpdateFulfillHTLC test_msg ""
 
 GEN_TEST ChannelAnnouncement test_msg_exact ""
 GEN_TEST NodeAnnouncement test_msg_exact ""
 GEN_TEST QueryShortChannelIds test_msg ""
-GEN_TEST ReplyShortChannelIdsEnd test_msg ""
-GEN_TEST QueryChannelRange test_msg ""
 GEN_TEST ReplyChannelRange test_msg ""
-GEN_TEST GossipTimestampFilter test_msg ""
 
-GEN_TEST UpdateAddHTLC test_msg_hole ", 85, 33"
 GEN_TEST ErrorMessage test_msg_hole ", 32, 2"
 GEN_TEST ChannelUpdate test_msg_hole ", 108, 1"
-
-GEN_TEST ClosingSigned test_msg_simple ""
-GEN_TEST Init test_msg_simple ""
-GEN_TEST OnionHopData test_msg_simple ""
-GEN_TEST Ping test_msg_simple ""
-GEN_TEST Pong test_msg_simple ""

fuzz/src/msg_targets/mod.rs

@@ -1,31 +1,31 @@
 mod utils;
 pub mod msg_accept_channel;
 pub mod msg_announcement_signatures;
-pub mod msg_channel_reestablish;
+pub mod msg_closing_signed;
 pub mod msg_commitment_signed;
-pub mod msg_decoded_onion_error_packet;
 pub mod msg_funding_created;
 pub mod msg_funding_locked;
 pub mod msg_funding_signed;
+pub mod msg_gossip_timestamp_filter;
+pub mod msg_init;
+pub mod msg_onion_hop_data;
 pub mod msg_open_channel;
+pub mod msg_ping;
+pub mod msg_pong;
+pub mod msg_query_channel_range;
+pub mod msg_reply_short_channel_ids_end;
 pub mod msg_revoke_and_ack;
 pub mod msg_shutdown;
+pub mod msg_update_add_htlc;
 pub mod msg_update_fail_htlc;
 pub mod msg_update_fail_malformed_htlc;
 pub mod msg_update_fee;
 pub mod msg_update_fulfill_htlc;
+pub mod msg_channel_reestablish;
+pub mod msg_decoded_onion_error_packet;
 pub mod msg_channel_announcement;
 pub mod msg_node_announcement;
 pub mod msg_query_short_channel_ids;
-pub mod msg_reply_short_channel_ids_end;
-pub mod msg_query_channel_range;
 pub mod msg_reply_channel_range;
-pub mod msg_gossip_timestamp_filter;
-pub mod msg_update_add_htlc;
 pub mod msg_error_message;
 pub mod msg_channel_update;
-pub mod msg_closing_signed;
-pub mod msg_init;
-pub mod msg_onion_hop_data;
-pub mod msg_ping;
-pub mod msg_pong;

fuzz/src/msg_targets/msg_accept_channel.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_accept_channel_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::AcceptChannel, data);
+	test_msg_simple!(msgs::AcceptChannel, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_accept_channel_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::AcceptChannel, data);
+	test_msg_simple!(msgs::AcceptChannel, data);
 }

fuzz/src/msg_targets/msg_announcement_signatures.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_announcement_signatures_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::AnnouncementSignatures, data);
+	test_msg_simple!(msgs::AnnouncementSignatures, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_announcement_signatures_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::AnnouncementSignatures, data);
+	test_msg_simple!(msgs::AnnouncementSignatures, data);
 }

fuzz/src/msg_targets/msg_commitment_signed.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_commitment_signed_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::CommitmentSigned, data);
+	test_msg_simple!(msgs::CommitmentSigned, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_commitment_signed_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::CommitmentSigned, data);
+	test_msg_simple!(msgs::CommitmentSigned, data);
 }

fuzz/src/msg_targets/msg_funding_created.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_funding_created_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::FundingCreated, data);
+	test_msg_simple!(msgs::FundingCreated, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_funding_created_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::FundingCreated, data);
+	test_msg_simple!(msgs::FundingCreated, data);
 }

fuzz/src/msg_targets/msg_funding_locked.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_funding_locked_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::FundingLocked, data);
+	test_msg_simple!(msgs::FundingLocked, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_funding_locked_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::FundingLocked, data);
+	test_msg_simple!(msgs::FundingLocked, data);
 }

fuzz/src/msg_targets/msg_funding_signed.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_funding_signed_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::FundingSigned, data);
+	test_msg_simple!(msgs::FundingSigned, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_funding_signed_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::FundingSigned, data);
+	test_msg_simple!(msgs::FundingSigned, data);
 }

fuzz/src/msg_targets/msg_gossip_timestamp_filter.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_gossip_timestamp_filter_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::GossipTimestampFilter, data);
+	test_msg_simple!(msgs::GossipTimestampFilter, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_gossip_timestamp_filter_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::GossipTimestampFilter, data);
+	test_msg_simple!(msgs::GossipTimestampFilter, data);
 }

fuzz/src/msg_targets/msg_open_channel.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_open_channel_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::OpenChannel, data);
+	test_msg_simple!(msgs::OpenChannel, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_open_channel_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::OpenChannel, data);
+	test_msg_simple!(msgs::OpenChannel, data);
 }

fuzz/src/msg_targets/msg_query_channel_range.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_query_channel_range_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::QueryChannelRange, data);
+	test_msg_simple!(msgs::QueryChannelRange, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_query_channel_range_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::QueryChannelRange, data);
+	test_msg_simple!(msgs::QueryChannelRange, data);
 }

fuzz/src/msg_targets/msg_reply_short_channel_ids_end.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_reply_short_channel_ids_end_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::ReplyShortChannelIdsEnd, data);
+	test_msg_simple!(msgs::ReplyShortChannelIdsEnd, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_reply_short_channel_ids_end_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::ReplyShortChannelIdsEnd, data);
+	test_msg_simple!(msgs::ReplyShortChannelIdsEnd, data);
 }

fuzz/src/msg_targets/msg_revoke_and_ack.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_revoke_and_ack_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::RevokeAndACK, data);
+	test_msg_simple!(msgs::RevokeAndACK, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_revoke_and_ack_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::RevokeAndACK, data);
+	test_msg_simple!(msgs::RevokeAndACK, data);
 }

fuzz/src/msg_targets/msg_shutdown.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_shutdown_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::Shutdown, data);
+	test_msg_simple!(msgs::Shutdown, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_shutdown_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::Shutdown, data);
+	test_msg_simple!(msgs::Shutdown, data);
 }

fuzz/src/msg_targets/msg_update_add_htlc.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_add_htlc_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg_hole!(msgs::UpdateAddHTLC, data, 85, 33);
+	test_msg_simple!(msgs::UpdateAddHTLC, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_add_htlc_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg_hole!(msgs::UpdateAddHTLC, data, 85, 33);
+	test_msg_simple!(msgs::UpdateAddHTLC, data);
 }

fuzz/src/msg_targets/msg_update_fail_htlc.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_fail_htlc_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::UpdateFailHTLC, data);
+	test_msg_simple!(msgs::UpdateFailHTLC, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_fail_htlc_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::UpdateFailHTLC, data);
+	test_msg_simple!(msgs::UpdateFailHTLC, data);
 }

fuzz/src/msg_targets/msg_update_fail_malformed_htlc.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_fail_malformed_htlc_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::UpdateFailMalformedHTLC, data);
+	test_msg_simple!(msgs::UpdateFailMalformedHTLC, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_fail_malformed_htlc_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::UpdateFailMalformedHTLC, data);
+	test_msg_simple!(msgs::UpdateFailMalformedHTLC, data);
 }

fuzz/src/msg_targets/msg_update_fee.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_fee_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::UpdateFee, data);
+	test_msg_simple!(msgs::UpdateFee, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_fee_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::UpdateFee, data);
+	test_msg_simple!(msgs::UpdateFee, data);
 }

fuzz/src/msg_targets/msg_update_fulfill_htlc.rs

@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_fulfill_htlc_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	test_msg!(msgs::UpdateFulfillHTLC, data);
+	test_msg_simple!(msgs::UpdateFulfillHTLC, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_fulfill_htlc_run(data: *const u8, datalen: usize) {
 	let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-	test_msg!(msgs::UpdateFulfillHTLC, data);
+	test_msg_simple!(msgs::UpdateFulfillHTLC, data);
 }