|
| 1 | +use crate::{hint, intrinsics, mem, ptr}; |
| 2 | + |
| 3 | +//#[rustc_const_stable_indirect] |
| 4 | +//#[rustc_allow_const_fn_unstable(const_eval_select)] |
| 5 | +#[rustc_const_unstable(feature = "const_swap_nonoverlapping", issue = "133668")] |
| 6 | +#[inline] |
| 7 | +pub(crate) const unsafe fn swap_nonoverlapping<T>(x: *mut T, y: *mut T, count: usize) { |
| 8 | + intrinsics::const_eval_select!( |
| 9 | + @capture[T] { x: *mut T, y: *mut T, count: usize }: |
| 10 | + if const { |
| 11 | + // At compile-time we want to always copy this in chunks of `T`, to ensure that if there |
| 12 | + // are pointers inside `T` we will copy them in one go rather than trying to copy a part |
| 13 | + // of a pointer (which would not work). |
| 14 | + // SAFETY: Same preconditions as this function |
| 15 | + unsafe { swap_nonoverlapping_const(x, y, count) } |
| 16 | + } else { |
| 17 | + // At runtime we want to make sure not to swap byte-for-byte for types like [u8; 15], |
| 18 | + // and swapping as `MaybeUninit<T>` doesn't actually work as untyped for things like |
| 19 | + // T = (u16, u8), so we type-erase to raw bytes and swap that way. |
| 20 | + // SAFETY: Same preconditions as this function |
| 21 | + unsafe { swap_nonoverlapping_runtime(x, y, count) } |
| 22 | + } |
| 23 | + ) |
| 24 | +} |
| 25 | + |
| 26 | +/// Same behavior and safety conditions as [`swap_nonoverlapping`] |
| 27 | +#[rustc_const_stable_indirect] |
| 28 | +#[inline] |
| 29 | +const unsafe fn swap_nonoverlapping_const<T>(x: *mut T, y: *mut T, count: usize) { |
| 30 | + let x = x.cast::<mem::MaybeUninit<T>>(); |
| 31 | + let y = y.cast::<mem::MaybeUninit<T>>(); |
| 32 | + let mut i = 0; |
| 33 | + while i < count { |
| 34 | + // SAFETY: By precondition, `i` is in-bounds because it's below `n` |
| 35 | + // and because the two input ranges are non-overlapping and read/writeable, |
| 36 | + // these individual items inside them are too. |
| 37 | + unsafe { |
| 38 | + intrinsics::untyped_swap_nonoverlapping::<T>(x.add(i), y.add(i)); |
| 39 | + } |
| 40 | + |
| 41 | + i += 1; |
| 42 | + } |
| 43 | +} |
| 44 | + |
| 45 | +// Scale the monomorphizations with the size of the machine, roughly. |
| 46 | +const MAX_ALIGN: usize = align_of::<usize>().pow(2); |
| 47 | + |
| 48 | +/// Same behavior and safety conditions as [`swap_nonoverlapping`] |
| 49 | +#[inline] |
| 50 | +unsafe fn swap_nonoverlapping_runtime<T>(x: *mut T, y: *mut T, count: usize) { |
| 51 | + let bytes = { |
| 52 | + let slice = ptr::slice_from_raw_parts(x, count); |
| 53 | + // SAFETY: Because they both exist in memory and don't overlap, they |
| 54 | + // must be legal slice sizes (below `isize::MAX` bytes). |
| 55 | + unsafe { mem::size_of_val_raw(slice) } |
| 56 | + }; |
| 57 | + |
| 58 | + // Generating *untyped* loops for every type is silly, so we polymorphize away |
| 59 | + // the actual type, but we want to take advantage of alignment if possible, |
| 60 | + // so monomorphize for a restricted set of possible alignments. |
| 61 | + macro_rules! delegate_by_alignment { |
| 62 | + ($($p:pat => $align:expr,)+) => {{ |
| 63 | + #![allow(unreachable_patterns)] |
| 64 | + match const { align_of::<T>() } { |
| 65 | + $( |
| 66 | + $p => { |
| 67 | + swap_nonoverlapping_bytes::<$align>(x.cast(), y.cast(), bytes); |
| 68 | + } |
| 69 | + )+ |
| 70 | + } |
| 71 | + }}; |
| 72 | + } |
| 73 | + |
| 74 | + // SAFETY: |
| 75 | + unsafe { |
| 76 | + delegate_by_alignment! { |
| 77 | + MAX_ALIGN.. => MAX_ALIGN, |
| 78 | + 64.. => 64, |
| 79 | + 32.. => 32, |
| 80 | + 16.. => 16, |
| 81 | + 8.. => 8, |
| 82 | + 4.. => 4, |
| 83 | + 2.. => 2, |
| 84 | + _ => 1, |
| 85 | + } |
| 86 | + } |
| 87 | +} |
| 88 | + |
| 89 | +/// # Safety: |
| 90 | +/// - `x` and `y` must be aligned to `ALIGN` |
| 91 | +/// - `bytes` must be a multiple of `ALIGN` |
| 92 | +/// - They must be readable, writable, and non-overlapping for `bytes` bytes |
| 93 | +#[inline] |
| 94 | +unsafe fn swap_nonoverlapping_bytes<const ALIGN: usize>( |
| 95 | + x: *mut mem::MaybeUninit<u8>, |
| 96 | + y: *mut mem::MaybeUninit<u8>, |
| 97 | + bytes: usize, |
| 98 | +) { |
| 99 | + // SAFETY: Two legal non-overlapping regions can't be bigger than this. |
| 100 | + // (And they couldn't have made allocations any bigger either anyway.) |
| 101 | + // FIXME: Would be nice to have a type for this instead of the assume. |
| 102 | + unsafe { hint::assert_unchecked(bytes < isize::MAX as usize) }; |
| 103 | + |
| 104 | + let mut i = 0; |
| 105 | + macro_rules! swap_next_n { |
| 106 | + ($n:expr) => {{ |
| 107 | + let x: *mut mem::MaybeUninit<[u8; $n]> = x.add(i).cast(); |
| 108 | + let y: *mut mem::MaybeUninit<[u8; $n]> = y.add(i).cast(); |
| 109 | + swap_nonoverlapping_aligned_chunk::<ALIGN, [u8; $n]>( |
| 110 | + x.as_mut_unchecked(), |
| 111 | + y.as_mut_unchecked(), |
| 112 | + ); |
| 113 | + i += $n; |
| 114 | + }}; |
| 115 | + } |
| 116 | + |
| 117 | + while bytes - i >= MAX_ALIGN { |
| 118 | + const { assert!(MAX_ALIGN >= ALIGN) }; |
| 119 | + // SAFETY: the const-assert above confirms we're only ever called with |
| 120 | + // an alignment equal to or smaller than max align, so this is necessarily |
| 121 | + // aligned, and the while loop ensures there's enough read/write memory. |
| 122 | + unsafe { |
| 123 | + swap_next_n!(MAX_ALIGN); |
| 124 | + } |
| 125 | + } |
| 126 | + |
| 127 | + macro_rules! handle_tail { |
| 128 | + ($($n:literal)+) => {$( |
| 129 | + if const { $n % ALIGN == 0 } { |
| 130 | + // Checking this way simplifies the block end to just add+test, |
| 131 | + // rather than needing extra math before the check. |
| 132 | + if (bytes & $n) != 0 { |
| 133 | + // SAFETY: The above swaps were bigger, so could not have |
| 134 | + // impacted the `$n`-relevant bit, so checking `bytes & $n` |
| 135 | + // was equivalent to `bytes - i >= $n`, and thus we have |
| 136 | + // enough space left to swap another `$n` bytes. |
| 137 | + unsafe { |
| 138 | + swap_next_n!($n); |
| 139 | + } |
| 140 | + } |
| 141 | + } |
| 142 | + )+}; |
| 143 | + } |
| 144 | + const { assert!(MAX_ALIGN <= 64) }; |
| 145 | + handle_tail!(32 16 8 4 2 1); |
| 146 | + |
| 147 | + debug_assert_eq!(i, bytes); |
| 148 | +} |
| 149 | + |
| 150 | +/// Swaps the `C` behind `x` and `y` as untyped memory |
| 151 | +/// |
| 152 | +/// # Safety |
| 153 | +/// |
| 154 | +/// Both `x` and `y` must be aligned to `ALIGN`, in addition to their normal alignment. |
| 155 | +/// They must be readable and writeable for `sizeof(C)` bytes, as usual for `&mut`s. |
| 156 | +/// |
| 157 | +/// (The actual instantiations are usually `C = [u8; _]`, so we get the alignment |
| 158 | +/// information from the loads by `assume`ing the passed-in alignment.) |
| 159 | +// Don't let MIR inline this, because we really want it to keep its noalias metadata |
| 160 | +#[rustc_no_mir_inline] |
| 161 | +#[inline] |
| 162 | +unsafe fn swap_nonoverlapping_aligned_chunk<const ALIGN: usize, C>( |
| 163 | + x: &mut mem::MaybeUninit<C>, |
| 164 | + y: &mut mem::MaybeUninit<C>, |
| 165 | +) { |
| 166 | + assert!(size_of::<C>() % ALIGN == 0); |
| 167 | + |
| 168 | + let x = ptr::from_mut(x); |
| 169 | + let y = ptr::from_mut(y); |
| 170 | + |
| 171 | + // SAFETY: One of our preconditions. |
| 172 | + unsafe { |
| 173 | + hint::assert_unchecked(x.is_aligned_to(ALIGN)); |
| 174 | + hint::assert_unchecked(y.is_aligned_to(ALIGN)); |
| 175 | + } |
| 176 | + |
| 177 | + // SAFETY: The memory is readable and writable because these were passed to |
| 178 | + // us as mutable references, and the untyped swap doesn't need validity. |
| 179 | + unsafe { |
| 180 | + intrinsics::untyped_swap_nonoverlapping::<C>(x, y); |
| 181 | + } |
| 182 | +} |
0 commit comments