Report allocation errors as panics

Author: Amanieu

Diff for: library/alloc/Cargo.toml

@@ -35,3 +35,6 @@ compiler-builtins-mem = ['compiler_builtins/mem']
 compiler-builtins-c = ["compiler_builtins/c"]
 compiler-builtins-no-asm = ["compiler_builtins/no-asm"]
 compiler-builtins-mangled-names = ["compiler_builtins/mangled-names"]
+
+# Make panics and failed asserts immediately abort without formatting any message
+panic_immediate_abort = ["core/panic_immediate_abort"]

Diff for: library/alloc/src/alloc.rs

@@ -14,6 +14,11 @@ use core::ptr::{self, NonNull};
 #[doc(inline)]
 pub use core::alloc::*;
 
+#[cfg(not(no_global_oom_handling))]
+use core::any::Any;
+#[cfg(not(no_global_oom_handling))]
+use core::panic::BoxMeUp;
+
 #[cfg(test)]
 mod tests;
 
@@ -343,14 +348,77 @@ pub(crate) unsafe fn box_free<T: ?Sized, A: Allocator>(ptr: Unique<T>, alloc: A)
     }
 }
 
-// # Allocation error handler
+/// Payload passed to the panic handler when `handle_alloc_error` is called.
+#[unstable(feature = "panic_oom_payload", issue = "none")]
+#[derive(Debug)]
+pub struct AllocErrorPanicPayload {
+    layout: Layout,
+}
+
+impl AllocErrorPanicPayload {
+    /// Internal function for the standard library to clone a payload.
+    #[unstable(feature = "std_internals", issue = "none")]
+    #[doc(hidden)]
+    pub fn internal_clone(&self) -> Self {
+        AllocErrorPanicPayload { layout: self.layout }
+    }
 
+    /// Returns the [`Layout`] of the allocation attempt that caused the error.
+    #[unstable(feature = "panic_oom_payload", issue = "none")]
+    pub fn layout(&self) -> Layout {
+        self.layout
+    }
+}
+
+#[unstable(feature = "std_internals", issue = "none")]
 #[cfg(not(no_global_oom_handling))]
-extern "Rust" {
-    // This is the magic symbol to call the global alloc error handler. rustc generates
-    // it to call `__rg_oom` if there is a `#[alloc_error_handler]`, or to call the
-    // default implementations below (`__rdl_oom`) otherwise.
-    fn __rust_alloc_error_handler(size: usize, align: usize) -> !;
+unsafe impl BoxMeUp for AllocErrorPanicPayload {
+    fn take_box(&mut self) -> *mut (dyn Any + Send) {
+        use crate::boxed::Box;
+        Box::into_raw(Box::new(self.internal_clone()))
+    }
+
+    fn get(&mut self) -> &(dyn Any + Send) {
+        self
+    }
+}
+
+// # Allocation error handler
+
+#[cfg(all(not(no_global_oom_handling), not(test)))]
+fn rust_oom(layout: Layout) -> ! {
+    if cfg!(feature = "panic_immediate_abort") {
+        core::intrinsics::abort()
+    }
+
+    extern "Rust" {
+        // NOTE This function never crosses the FFI boundary; it's a Rust-to-Rust call
+        // that gets resolved to the `#[panic_handler]` function.
+        #[lang = "panic_impl"]
+        fn panic_impl(pi: &core::panic::PanicInfo<'_>) -> !;
+
+        // This symbol is emitted by rustc next to __rust_alloc_error_handler.
+        // Its value depends on the -Zoom={panic,abort} compiler option.
+        static __rust_alloc_error_handler_should_panic: u8;
+    }
+
+    // Hack to work around issues with the lifetime of Arguments.
+    match format_args!("memory allocation of {} bytes failed", layout.size()) {
+        fmt => {
+            // Create a PanicInfo with a custom payload for the panic handler.
+            let can_unwind = unsafe { __rust_alloc_error_handler_should_panic != 0 };
+            let mut pi = core::panic::PanicInfo::internal_constructor(
+                Some(&fmt),
+                core::panic::Location::caller(),
+                can_unwind,
+            );
+            let payload = AllocErrorPanicPayload { layout };
+            pi.set_payload(&payload);
+
+            // SAFETY: `panic_impl` is defined in safe Rust code and thus is safe to call.
+            unsafe { panic_impl(&pi) }
+        }
+    }
 }
 
 /// Abort on memory allocation error or failure.
@@ -375,9 +443,7 @@ pub const fn handle_alloc_error(layout: Layout) -> ! {
     }
 
     fn rt_error(layout: Layout) -> ! {
-        unsafe {
-            __rust_alloc_error_handler(layout.size(), layout.align());
-        }
+        rust_oom(layout);
     }
 
     unsafe { core::intrinsics::const_eval_select((layout,), ct_error, rt_error) }

Diff for: library/alloc/src/lib.rs

@@ -136,6 +136,7 @@
 #![feature(maybe_uninit_slice)]
 #![feature(maybe_uninit_uninit_array)]
 #![feature(maybe_uninit_uninit_array_transpose)]
+#![feature(panic_internals)]
 #![feature(pattern)]
 #![feature(pointer_byte_offsets)]
 #![feature(provide_any)]

Diff for: library/std/Cargo.toml

@@ -67,7 +67,7 @@ llvm-libunwind = ["unwind/llvm-libunwind"]
 system-llvm-libunwind = ["unwind/system-llvm-libunwind"]
 
 # Make panics and failed asserts immediately abort without formatting any message
-panic_immediate_abort = ["core/panic_immediate_abort"]
+panic_immediate_abort = ["alloc/panic_immediate_abort"]
 
 # Enable std_detect default features for stdarch/crates/std_detect:
 # https://github.com/rust-lang/stdarch/blob/master/crates/std_detect/Cargo.toml

Diff for: library/std/src/lib.rs

@@ -319,6 +319,7 @@
 #![feature(get_mut_unchecked)]
 #![feature(map_try_insert)]
 #![feature(new_uninit)]
+#![feature(panic_oom_payload)]
 #![feature(slice_concat_trait)]
 #![feature(thin_box)]
 #![feature(try_reserve_kind)]

Diff for: library/std/src/panicking.rs

@@ -245,19 +245,24 @@ fn default_hook(info: &PanicInfo<'_>) {
 
     // The current implementation always returns `Some`.
     let location = info.location().unwrap();
-
-    let msg = match info.payload().downcast_ref::<&'static str>() {
-        Some(s) => *s,
-        None => match info.payload().downcast_ref::<String>() {
-            Some(s) => &s[..],
-            None => "Box<dyn Any>",
-        },
-    };
     let thread = thread_info::current_thread();
     let name = thread.as_ref().and_then(|t| t.name()).unwrap_or("<unnamed>");
 
     let write = |err: &mut dyn crate::io::Write| {
-        let _ = writeln!(err, "thread '{name}' panicked at '{msg}', {location}");
+        // Use the panic message directly if available, otherwise take it from
+        // the payload.
+        if let Some(msg) = info.message() {
+            let _ = writeln!(err, "thread '{name}' panicked at '{msg}', {location}");
+        } else {
+            let msg = if let Some(s) = info.payload().downcast_ref::<&'static str>() {
+                *s
+            } else if let Some(s) = info.payload().downcast_ref::<String>() {
+                &s[..]
+            } else {
+                "Box<dyn Any>"
+            };
+            let _ = writeln!(err, "thread '{name}' panicked at '{msg}', {location}");
+        }
 
         static FIRST_PANIC: AtomicBool = AtomicBool::new(true);
 
@@ -524,6 +529,8 @@ pub fn panicking() -> bool {
 #[cfg(not(test))]
 #[panic_handler]
 pub fn begin_panic_handler(info: &PanicInfo<'_>) -> ! {
+    use alloc::alloc::AllocErrorPanicPayload;
+
     struct PanicPayload<'a> {
         inner: &'a fmt::Arguments<'a>,
         string: Option<String>,
@@ -550,8 +557,7 @@ pub fn begin_panic_handler(info: &PanicInfo<'_>) -> ! {
     unsafe impl<'a> BoxMeUp for PanicPayload<'a> {
         fn take_box(&mut self) -> *mut (dyn Any + Send) {
             // We do two allocations here, unfortunately. But (a) they're required with the current
-            // scheme, and (b) we don't handle panic + OOM properly anyway (see comment in
-            // begin_panic below).
+            // scheme, and (b) OOM uses its own separate payload type which doesn't allocate.
             let contents = mem::take(self.fill());
             Box::into_raw(Box::new(contents))
         }
@@ -576,7 +582,14 @@ pub fn begin_panic_handler(info: &PanicInfo<'_>) -> ! {
     let loc = info.location().unwrap(); // The current implementation always returns Some
     let msg = info.message().unwrap(); // The current implementation always returns Some
     crate::sys_common::backtrace::__rust_end_short_backtrace(move || {
-        if let Some(msg) = msg.as_str() {
+        if let Some(payload) = info.payload().downcast_ref::<AllocErrorPanicPayload>() {
+            rust_panic_with_hook(
+                &mut payload.internal_clone(),
+                info.message(),
+                loc,
+                info.can_unwind(),
+            );
+        } else if let Some(msg) = msg.as_str() {
             rust_panic_with_hook(&mut StrPanicPayload(msg), info.message(), loc, info.can_unwind());
         } else {
             rust_panic_with_hook(
@@ -623,11 +636,7 @@ pub const fn begin_panic<M: Any + Send>(msg: M) -> ! {
 
     unsafe impl<A: Send + 'static> BoxMeUp for PanicPayload<A> {
         fn take_box(&mut self) -> *mut (dyn Any + Send) {
-            // Note that this should be the only allocation performed in this code path. Currently
-            // this means that panic!() on OOM will invoke this code path, but then again we're not
-            // really ready for panic on OOM anyway. If we do start doing this, then we should
-            // propagate this allocation to be performed in the parent of this thread instead of the
-            // thread that's panicking.
+            // Note that this should be the only allocation performed in this code path.
             let data = match self.inner.take() {
                 Some(a) => Box::new(a) as Box<dyn Any + Send>,
                 None => process::abort(),