perf: Compute ctrl_align statically per T again

Author: Markus Westerlind

src/raw/mod.rs

@@ -240,6 +240,41 @@ fn calculate_layout<T>(buckets: usize) -> Option<(Layout, usize)> {
     data.extend(ctrl).ok()
 }
 
+/// Helper which allows the max calculation for ctrl_align to be statically computed for each T
+/// while keeping the rest of `calculate_layout_for` independent of `T`
+#[derive(Copy, Clone)]
+struct TableLayout {
+    size: usize,
+    ctrl_align: usize,
+}
+
+impl TableLayout {
+    #[inline]
+    fn new<T>() -> Self {
+        let layout = Layout::new::<T>();
+        Self {
+            size: layout.size(),
+            ctrl_align: usize::max(layout.align(), Group::WIDTH),
+        }
+    }
+
+    #[inline]
+    fn calculate_layout_for(self, buckets: usize) -> Option<(Layout, usize)> {
+        debug_assert!(buckets.is_power_of_two());
+
+        let TableLayout { size, ctrl_align } = self;
+        // Manual layout calculation since Layout methods are not yet stable.
+        let ctrl_offset =
+            size.checked_mul(buckets)?.checked_add(ctrl_align - 1)? & !(ctrl_align - 1);
+        let len = ctrl_offset.checked_add(buckets + Group::WIDTH)?;
+
+        Some((
+            unsafe { Layout::from_size_align_unchecked(len, ctrl_align) },
+            ctrl_offset,
+        ))
+    }
+}
+
 /// Returns a Layout which describes the allocation required for a hash table,
 /// and the offset of the control bytes in the allocation.
 /// (the offset is also one past last element of buckets)
@@ -248,26 +283,7 @@ fn calculate_layout<T>(buckets: usize) -> Option<(Layout, usize)> {
 #[cfg_attr(feature = "inline-more", inline)]
 #[cfg(not(feature = "nightly"))]
 fn calculate_layout<T>(buckets: usize) -> Option<(Layout, usize)> {
-    calculate_layout_for(Layout::new::<T>(), buckets)
-}
-
-#[inline]
-fn calculate_layout_for(layout: Layout, buckets: usize) -> Option<(Layout, usize)> {
-    debug_assert!(buckets.is_power_of_two());
-
-    // Manual layout calculation since Layout methods are not yet stable.
-    let ctrl_align = usize::max(layout.align(), Group::WIDTH);
-    let ctrl_offset = layout
-        .size()
-        .checked_mul(buckets)?
-        .checked_add(ctrl_align - 1)?
-        & !(ctrl_align - 1);
-    let len = ctrl_offset.checked_add(buckets + Group::WIDTH)?;
-
-    Some((
-        unsafe { Layout::from_size_align_unchecked(len, ctrl_align) },
-        ctrl_offset,
-    ))
+    TableLayout::new::<T>().calculate_layout_for(buckets)
 }
 
 /// A reference to a hash table bucket containing a `T`.
@@ -411,7 +427,7 @@ impl<T> RawTable<T> {
         debug_assert!(buckets.is_power_of_two());
 
         Ok(Self {
-            table: RawTableInner::new_uninitialized(Layout::new::<T>(), buckets, fallability)?,
+            table: RawTableInner::new_uninitialized(TableLayout::new::<T>(), buckets, fallability)?,
             marker: PhantomData,
         })
     }
@@ -424,7 +440,7 @@ impl<T> RawTable<T> {
     ) -> Result<Self, TryReserveError> {
         Ok(Self {
             table: RawTableInner::fallible_with_capacity(
-                Layout::new::<T>(),
+                TableLayout::new::<T>(),
                 capacity,
                 fallability,
             )?,
@@ -452,7 +468,7 @@ impl<T> RawTable<T> {
     /// Deallocates the table without dropping any entries.
     #[cfg_attr(feature = "inline-more", inline)]
     unsafe fn free_buckets(&mut self) {
-        self.table.free_buckets(Layout::new::<T>())
+        self.table.free_buckets(TableLayout::new::<T>())
     }
 
     /// Returns pointer to one past last element of data table.
@@ -719,7 +735,7 @@ impl<T> RawTable<T> {
         unsafe {
             let mut new_table =
                 self.table
-                    .prepare_resize(Layout::new::<T>(), capacity, fallability)?;
+                    .prepare_resize(TableLayout::new::<T>(), capacity, fallability)?;
 
             // Copy all elements to the new table.
             for item in self.iter() {
@@ -988,14 +1004,14 @@ impl RawTableInner {
 
     #[inline]
     unsafe fn new_uninitialized(
-        t_layout: Layout,
+        table_layout: TableLayout,
         buckets: usize,
         fallability: Fallibility,
     ) -> Result<Self, TryReserveError> {
         debug_assert!(buckets.is_power_of_two());
 
         // Avoid `Option::ok_or_else` because it bloats LLVM IR.
-        let (layout, ctrl_offset) = match calculate_layout_for(t_layout, buckets) {
+        let (layout, ctrl_offset) = match table_layout.calculate_layout_for(buckets) {
             Some(lco) => lco,
             None => return Err(fallability.capacity_overflow()),
         };
@@ -1016,7 +1032,7 @@ impl RawTableInner {
 
     #[inline]
     fn fallible_with_capacity(
-        t_layout: Layout,
+        table_layout: TableLayout,
         capacity: usize,
         fallability: Fallibility,
     ) -> Result<Self, TryReserveError> {
@@ -1027,7 +1043,7 @@ impl RawTableInner {
                 let buckets =
                     capacity_to_buckets(capacity).ok_or_else(|| fallability.capacity_overflow())?;
 
-                let result = Self::new_uninitialized(t_layout, buckets, fallability)?;
+                let result = Self::new_uninitialized(table_layout, buckets, fallability)?;
                 result.ctrl(0).write_bytes(EMPTY, result.num_ctrl_bytes());
 
                 Ok(result)
@@ -1261,14 +1277,15 @@ impl RawTableInner {
     #[inline]
     unsafe fn prepare_resize(
         &self,
-        layout_t: Layout,
+        table_layout: TableLayout,
         capacity: usize,
         fallability: Fallibility,
     ) -> Result<crate::scopeguard::ScopeGuard<Self, impl FnMut(&mut Self)>, TryReserveError> {
         debug_assert!(self.items <= capacity);
 
         // Allocate and initialize the new table.
-        let mut new_table = RawTableInner::fallible_with_capacity(layout_t, capacity, fallability)?;
+        let mut new_table =
+            RawTableInner::fallible_with_capacity(table_layout, capacity, fallability)?;
         new_table.growth_left -= self.items;
         new_table.items = self.items;
 
@@ -1280,15 +1297,15 @@ impl RawTableInner {
         // the comment at the bottom of this function.
         Ok(guard(new_table, move |self_| {
             if !self_.is_empty_singleton() {
-                self_.free_buckets(layout_t);
+                self_.free_buckets(table_layout);
             }
         }))
     }
 
     #[inline]
-    unsafe fn free_buckets(&mut self, t_layout: Layout) {
+    unsafe fn free_buckets(&mut self, table_layout: TableLayout) {
         // Avoid `Option::unwrap_or_else` because it bloats LLVM IR.
-        let (layout, ctrl_offset) = match calculate_layout_for(t_layout, self.buckets()) {
+        let (layout, ctrl_offset) = match table_layout.calculate_layout_for(self.buckets()) {
             Some(lco) => lco,
             None => hint::unreachable_unchecked(),
         };