Skip to content

Commit 076ef66

Browse files
TimDiekmannTimDiekmann
committed
Remove in-place allocation and revert to separate methods for zeroed allocations
Fix docs
1 parent 1f5d69d commit 076ef66

File tree

14 files changed

+393
-350
lines changed

14 files changed

+393
-350
lines changed

‎library/alloc/src/alloc.rs

+69-43
Original file line numberDiff line numberDiff line change
@@ -164,16 +164,26 @@ pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 {
164164
#[unstable(feature = "allocator_api", issue = "32838")]
165165
unsafe impl AllocRef for Global {
166166
#[inline]
167-
fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result<MemoryBlock, AllocErr> {
167+
fn alloc(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
168168
unsafe {
169169
let size = layout.size();
170170
if size == 0 {
171171
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
172172
} else {
173-
let raw_ptr = match init {
174-
AllocInit::Uninitialized => alloc(layout),
175-
AllocInit::Zeroed => alloc_zeroed(layout),
176-
};
173+
let raw_ptr = alloc(layout);
174+
let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
175+
Ok(MemoryBlock { ptr, size })
176+
}
177+
}
178+
}
179+
180+
fn alloc_zeroed(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
181+
unsafe {
182+
let size = layout.size();
183+
if size == 0 {
184+
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
185+
} else {
186+
let raw_ptr = alloc_zeroed(layout);
177187
let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
178188
Ok(MemoryBlock { ptr, size })
179189
}
@@ -193,8 +203,6 @@ unsafe impl AllocRef for Global {
193203
ptr: NonNull<u8>,
194204
layout: Layout,
195205
new_size: usize,
196-
placement: ReallocPlacement,
197-
init: AllocInit,
198206
) -> Result<MemoryBlock, AllocErr> {
199207
let size = layout.size();
200208
debug_assert!(
@@ -206,26 +214,49 @@ unsafe impl AllocRef for Global {
206214
return Ok(MemoryBlock { ptr, size });
207215
}
208216

209-
match placement {
210-
ReallocPlacement::InPlace => Err(AllocErr),
211-
ReallocPlacement::MayMove if layout.size() == 0 => {
212-
let new_layout =
213-
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
214-
self.alloc(new_layout, init)
215-
}
216-
ReallocPlacement::MayMove => {
217-
// `realloc` probably checks for `new_size > size` or something similar.
218-
let ptr = unsafe {
219-
intrinsics::assume(new_size > size);
220-
realloc(ptr.as_ptr(), layout, new_size)
221-
};
222-
let memory =
223-
MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size };
224-
unsafe {
225-
init.init_offset(memory, size);
226-
}
227-
Ok(memory)
217+
if layout.size() == 0 {
218+
let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
219+
self.alloc(new_layout)
220+
} else {
221+
// `realloc` probably checks for `new_size > size` or something similar.
222+
let ptr = unsafe {
223+
intrinsics::assume(new_size > size);
224+
realloc(ptr.as_ptr(), layout, new_size)
225+
};
226+
Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size })
227+
}
228+
}
229+
230+
unsafe fn grow_zeroed(
231+
&mut self,
232+
ptr: NonNull<u8>,
233+
layout: Layout,
234+
new_size: usize,
235+
) -> Result<MemoryBlock, AllocErr> {
236+
let size = layout.size();
237+
debug_assert!(
238+
new_size >= size,
239+
"`new_size` must be greater than or equal to `memory.size()`"
240+
);
241+
242+
if size == new_size {
243+
return Ok(MemoryBlock { ptr, size });
244+
}
245+
246+
if layout.size() == 0 {
247+
let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
248+
self.alloc(new_layout)
249+
} else {
250+
// `realloc` probably checks for `new_size > size` or something similar.
251+
let ptr = unsafe {
252+
intrinsics::assume(new_size > size);
253+
realloc(ptr.as_ptr(), layout, new_size)
254+
};
255+
let memory = MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size };
256+
unsafe {
257+
memory.ptr.as_ptr().add(size).write_bytes(0, memory.size - size);
228258
}
259+
Ok(memory)
229260
}
230261
}
231262

@@ -235,7 +266,6 @@ unsafe impl AllocRef for Global {
235266
ptr: NonNull<u8>,
236267
layout: Layout,
237268
new_size: usize,
238-
placement: ReallocPlacement,
239269
) -> Result<MemoryBlock, AllocErr> {
240270
let size = layout.size();
241271
debug_assert!(
@@ -247,22 +277,18 @@ unsafe impl AllocRef for Global {
247277
return Ok(MemoryBlock { ptr, size });
248278
}
249279

250-
match placement {
251-
ReallocPlacement::InPlace => Err(AllocErr),
252-
ReallocPlacement::MayMove if new_size == 0 => {
253-
unsafe {
254-
self.dealloc(ptr, layout);
255-
}
256-
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
257-
}
258-
ReallocPlacement::MayMove => {
259-
// `realloc` probably checks for `new_size < size` or something similar.
260-
let ptr = unsafe {
261-
intrinsics::assume(new_size < size);
262-
realloc(ptr.as_ptr(), layout, new_size)
263-
};
264-
Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size })
280+
if new_size == 0 {
281+
unsafe {
282+
self.dealloc(ptr, layout);
265283
}
284+
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
285+
} else {
286+
// `realloc` probably checks for `new_size < size` or something similar.
287+
let ptr = unsafe {
288+
intrinsics::assume(new_size < size);
289+
realloc(ptr.as_ptr(), layout, new_size)
290+
};
291+
Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size })
266292
}
267293
}
268294
}
@@ -274,7 +300,7 @@ unsafe impl AllocRef for Global {
274300
#[inline]
275301
unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 {
276302
let layout = unsafe { Layout::from_size_align_unchecked(size, align) };
277-
match Global.alloc(layout, AllocInit::Uninitialized) {
303+
match Global.alloc(layout) {
278304
Ok(memory) => memory.ptr.as_ptr(),
279305
Err(_) => handle_alloc_error(layout),
280306
}

‎library/alloc/src/alloc/tests.rs

+2-3
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,8 @@ use test::Bencher;
88
fn allocate_zeroed() {
99
unsafe {
1010
let layout = Layout::from_size_align(1024, 1).unwrap();
11-
let memory = Global
12-
.alloc(layout.clone(), AllocInit::Zeroed)
13-
.unwrap_or_else(|_| handle_alloc_error(layout));
11+
let memory =
12+
Global.alloc_zeroed(layout.clone()).unwrap_or_else(|_| handle_alloc_error(layout));
1413

1514
let mut i = memory.ptr.cast::<u8>().as_ptr();
1615
let end = i.add(layout.size());

‎library/alloc/src/boxed.rs

+4-7
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ use core::pin::Pin;
146146
use core::ptr::{self, NonNull, Unique};
147147
use core::task::{Context, Poll};
148148

149-
use crate::alloc::{self, AllocInit, AllocRef, Global};
149+
use crate::alloc::{self, AllocRef, Global};
150150
use crate::borrow::Cow;
151151
use crate::raw_vec::RawVec;
152152
use crate::str::from_boxed_utf8_unchecked;
@@ -197,11 +197,8 @@ impl<T> Box<T> {
197197
#[unstable(feature = "new_uninit", issue = "63291")]
198198
pub fn new_uninit() -> Box<mem::MaybeUninit<T>> {
199199
let layout = alloc::Layout::new::<mem::MaybeUninit<T>>();
200-
let ptr = Global
201-
.alloc(layout, AllocInit::Uninitialized)
202-
.unwrap_or_else(|_| alloc::handle_alloc_error(layout))
203-
.ptr
204-
.cast();
200+
let ptr =
201+
Global.alloc(layout).unwrap_or_else(|_| alloc::handle_alloc_error(layout)).ptr.cast();
205202
unsafe { Box::from_raw(ptr.as_ptr()) }
206203
}
207204

@@ -227,7 +224,7 @@ impl<T> Box<T> {
227224
pub fn new_zeroed() -> Box<mem::MaybeUninit<T>> {
228225
let layout = alloc::Layout::new::<mem::MaybeUninit<T>>();
229226
let ptr = Global
230-
.alloc(layout, AllocInit::Zeroed)
227+
.alloc_zeroed(layout)
231228
.unwrap_or_else(|_| alloc::handle_alloc_error(layout))
232229
.ptr
233230
.cast();

‎library/alloc/src/raw_vec.rs

+22-22
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,20 @@ use core::ops::Drop;
88
use core::ptr::{NonNull, Unique};
99
use core::slice;
1010

11-
use crate::alloc::{
12-
handle_alloc_error,
13-
AllocInit::{self, *},
14-
AllocRef, Global, Layout,
15-
ReallocPlacement::{self, *},
16-
};
11+
use crate::alloc::{handle_alloc_error, AllocRef, Global, Layout};
1712
use crate::boxed::Box;
1813
use crate::collections::TryReserveError::{self, *};
1914

2015
#[cfg(test)]
2116
mod tests;
2217

18+
enum AllocInit {
19+
/// The contents of the new memory are uninitialized.
20+
Uninitialized,
21+
/// The new memory is guaranteed to be zeroed.
22+
Zeroed,
23+
}
24+
2325
/// A low-level utility for more ergonomically allocating, reallocating, and deallocating
2426
/// a buffer of memory on the heap without having to worry about all the corner cases
2527
/// involved. This type is excellent for building your own data structures like Vec and VecDeque.
@@ -156,14 +158,14 @@ impl<T, A: AllocRef> RawVec<T, A> {
156158
/// allocator for the returned `RawVec`.
157159
#[inline]
158160
pub fn with_capacity_in(capacity: usize, alloc: A) -> Self {
159-
Self::allocate_in(capacity, Uninitialized, alloc)
161+
Self::allocate_in(capacity, AllocInit::Uninitialized, alloc)
160162
}
161163

162164
/// Like `with_capacity_zeroed`, but parameterized over the choice
163165
/// of allocator for the returned `RawVec`.
164166
#[inline]
165167
pub fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self {
166-
Self::allocate_in(capacity, Zeroed, alloc)
168+
Self::allocate_in(capacity, AllocInit::Zeroed, alloc)
167169
}
168170

169171
fn allocate_in(capacity: usize, init: AllocInit, mut alloc: A) -> Self {
@@ -180,7 +182,11 @@ impl<T, A: AllocRef> RawVec<T, A> {
180182
Ok(_) => {}
181183
Err(_) => capacity_overflow(),
182184
}
183-
let memory = match alloc.alloc(layout, init) {
185+
let result = match init {
186+
AllocInit::Uninitialized => alloc.alloc(layout),
187+
AllocInit::Zeroed => alloc.alloc_zeroed(layout),
188+
};
189+
let memory = match result {
184190
Ok(memory) => memory,
185191
Err(_) => handle_alloc_error(layout),
186192
};
@@ -358,7 +364,7 @@ impl<T, A: AllocRef> RawVec<T, A> {
358364
///
359365
/// Aborts on OOM.
360366
pub fn shrink_to_fit(&mut self, amount: usize) {
361-
match self.shrink(amount, MayMove) {
367+
match self.shrink(amount) {
362368
Err(CapacityOverflow) => capacity_overflow(),
363369
Err(AllocError { layout, .. }) => handle_alloc_error(layout),
364370
Ok(()) => { /* yay */ }
@@ -450,22 +456,16 @@ impl<T, A: AllocRef> RawVec<T, A> {
450456
Ok(())
451457
}
452458

453-
fn shrink(
454-
&mut self,
455-
amount: usize,
456-
placement: ReallocPlacement,
457-
) -> Result<(), TryReserveError> {
459+
fn shrink(&mut self, amount: usize) -> Result<(), TryReserveError> {
458460
assert!(amount <= self.capacity(), "Tried to shrink to a larger capacity");
459461

460462
let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) };
461463
let new_size = amount * mem::size_of::<T>();
462464

463465
let memory = unsafe {
464-
self.alloc.shrink(ptr, layout, new_size, placement).map_err(|_| {
465-
TryReserveError::AllocError {
466-
layout: Layout::from_size_align_unchecked(new_size, layout.align()),
467-
non_exhaustive: (),
468-
}
466+
self.alloc.shrink(ptr, layout, new_size).map_err(|_| TryReserveError::AllocError {
467+
layout: Layout::from_size_align_unchecked(new_size, layout.align()),
468+
non_exhaustive: (),
469469
})?
470470
};
471471
self.set_memory(memory);
@@ -492,9 +492,9 @@ where
492492

493493
let memory = if let Some((ptr, old_layout)) = current_memory {
494494
debug_assert_eq!(old_layout.align(), new_layout.align());
495-
unsafe { alloc.grow(ptr, old_layout, new_layout.size(), MayMove, Uninitialized) }
495+
unsafe { alloc.grow(ptr, old_layout, new_layout.size()) }
496496
} else {
497-
alloc.alloc(new_layout, Uninitialized)
497+
alloc.alloc(new_layout)
498498
}
499499
.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })?;
500500

‎library/alloc/src/raw_vec/tests.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ fn allocator_param() {
2020
fuel: usize,
2121
}
2222
unsafe impl AllocRef for BoundedAlloc {
23-
fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result<MemoryBlock, AllocErr> {
23+
fn alloc(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
2424
let size = layout.size();
2525
if size > self.fuel {
2626
return Err(AllocErr);
2727
}
28-
match Global.alloc(layout, init) {
28+
match Global.alloc(layout) {
2929
ok @ Ok(_) => {
3030
self.fuel -= size;
3131
ok

‎library/alloc/src/rc.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ use core::pin::Pin;
250250
use core::ptr::{self, NonNull};
251251
use core::slice::from_raw_parts_mut;
252252

253-
use crate::alloc::{box_free, handle_alloc_error, AllocInit, AllocRef, Global, Layout};
253+
use crate::alloc::{box_free, handle_alloc_error, AllocRef, Global, Layout};
254254
use crate::borrow::{Cow, ToOwned};
255255
use crate::string::String;
256256
use crate::vec::Vec;
@@ -951,9 +951,7 @@ impl<T: ?Sized> Rc<T> {
951951
let layout = Layout::new::<RcBox<()>>().extend(value_layout).unwrap().0.pad_to_align();
952952

953953
// Allocate for the layout.
954-
let mem = Global
955-
.alloc(layout, AllocInit::Uninitialized)
956-
.unwrap_or_else(|_| handle_alloc_error(layout));
954+
let mem = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout));
957955

958956
// Initialize the RcBox
959957
let inner = mem_to_rcbox(mem.ptr.as_ptr());

‎library/alloc/src/sync.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use core::slice::from_raw_parts_mut;
2323
use core::sync::atomic;
2424
use core::sync::atomic::Ordering::{Acquire, Relaxed, Release, SeqCst};
2525

26-
use crate::alloc::{box_free, handle_alloc_error, AllocInit, AllocRef, Global, Layout};
26+
use crate::alloc::{box_free, handle_alloc_error, AllocRef, Global, Layout};
2727
use crate::borrow::{Cow, ToOwned};
2828
use crate::boxed::Box;
2929
use crate::rc::is_dangling;
@@ -906,9 +906,7 @@ impl<T: ?Sized> Arc<T> {
906906
// reference (see #54908).
907907
let layout = Layout::new::<ArcInner<()>>().extend(value_layout).unwrap().0.pad_to_align();
908908

909-
let mem = Global
910-
.alloc(layout, AllocInit::Uninitialized)
911-
.unwrap_or_else(|_| handle_alloc_error(layout));
909+
let mem = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout));
912910

913911
// Initialize the ArcInner
914912
let inner = mem_to_arcinner(mem.ptr.as_ptr());

‎library/alloc/tests/heap.rs

+2-8
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use std::alloc::{AllocInit, AllocRef, Global, Layout, System};
1+
use std::alloc::{AllocRef, Global, Layout, System};
22

33
/// Issue #45955 and #62251.
44
#[test]
@@ -20,13 +20,7 @@ fn check_overalign_requests<T: AllocRef>(mut allocator: T) {
2020
unsafe {
2121
let pointers: Vec<_> = (0..iterations)
2222
.map(|_| {
23-
allocator
24-
.alloc(
25-
Layout::from_size_align(size, align).unwrap(),
26-
AllocInit::Uninitialized,
27-
)
28-
.unwrap()
29-
.ptr
23+
allocator.alloc(Layout::from_size_align(size, align).unwrap()).unwrap().ptr
3024
})
3125
.collect();
3226
for &ptr in &pointers {

0 commit comments

Comments
 (0)