Skip to content

Commit 3f7f5e8

Browse files
authored
Optimize RcInnerPtr::inc_strong instruction count
Inspired by this internals thread: https://internals.rust-lang.org/t/rc-optimization-on-64-bit-targets/16362 [The generated assembly is a bit smaller](https://rust.godbolt.org/z/TeTnf6144) and is a more efficient usage of the CPU's instruction cache. `unlikely` doesn't impact any of the small artificial tests I've done, but I've included it in case it might help more complex scenarios when this is inlined.
1 parent a4a5e79 commit 3f7f5e8

File tree

1 file changed

+24
-10
lines changed

1 file changed

+24
-10
lines changed

library/alloc/src/rc.rs

+24-10
Original file line numberDiff line numberDiff line change
@@ -2512,14 +2512,21 @@ trait RcInnerPtr {
25122512
fn inc_strong(&self) {
25132513
let strong = self.strong();
25142514

2515+
// We insert an `assume` here to hint LLVM at an otherwise
2516+
// missed optimization.
2517+
// SAFETY: The reference count will never be zero when this is
2518+
// called.
2519+
unsafe { core::intrinsics::assume(strong != 0); }
2520+
2521+
let strong = strong.wrapping_add(1);
2522+
self.strong_ref().set(strong);
2523+
25152524
// We want to abort on overflow instead of dropping the value.
2516-
// The reference count will never be zero when this is called;
2517-
// nevertheless, we insert an abort here to hint LLVM at
2518-
// an otherwise missed optimization.
2519-
if strong == 0 || strong == usize::MAX {
2525+
// Checking after the store instead of before allows for
2526+
// slightly better code generation.
2527+
if core::intrinsics::unlikely(strong == 0) {
25202528
abort();
25212529
}
2522-
self.strong_ref().set(strong + 1);
25232530
}
25242531

25252532
#[inline]
@@ -2536,14 +2543,21 @@ trait RcInnerPtr {
25362543
fn inc_weak(&self) {
25372544
let weak = self.weak();
25382545

2546+
// We insert an `assume` here to hint LLVM at an otherwise
2547+
// missed optimization.
2548+
// SAFETY: The reference count will never be zero when this is
2549+
// called.
2550+
unsafe { core::intrinsics::assume(weak != 0); }
2551+
2552+
let weak = weak.wrapping_add(1);
2553+
self.weak_ref().set(weak);
2554+
25392555
// We want to abort on overflow instead of dropping the value.
2540-
// The reference count will never be zero when this is called;
2541-
// nevertheless, we insert an abort here to hint LLVM at
2542-
// an otherwise missed optimization.
2543-
if weak == 0 || weak == usize::MAX {
2556+
// Checking after the store instead of before allows for
2557+
// slightly better code generation.
2558+
if core::intrinsics::unlikely(weak == 0) {
25442559
abort();
25452560
}
2546-
self.weak_ref().set(weak + 1);
25472561
}
25482562

25492563
#[inline]

0 commit comments

Comments
 (0)