Abort when a GC thread panics.

Previously the behavior is printing backtrace and terminating the GC
thread itself instead of the entire process, and the mutator threads
will be hanging there waiting for GC to finish.

Author: wks

mmtk/src/api.rs

@@ -79,6 +79,8 @@ pub extern "C" fn mmtk_init_binding(
     binding_options: *const RubyBindingOptions,
     upcalls: *const abi::RubyUpcalls,
 ) {
+    crate::set_panic_hook();
+
     let builder = unsafe { Box::from_raw(builder) };
     let binding_options = unsafe { &*binding_options };
     let mmtk_boxed = mmtk_init(&builder);

mmtk/src/collection.rs

@@ -35,6 +35,7 @@ impl Collection<Ruby> for VMCollection {
                     .name("MMTk Controller Thread".to_string())
                     .spawn(move || {
                         debug!("Hello! This is MMTk Controller Thread running!");
+                        crate::register_gc_thread(thread::current().id());
                         let ptr_controller = &mut *controller as *mut GCController<Ruby>;
                         let gc_thread_tls =
                             Box::into_raw(Box::new(GCThreadTLS::for_controller(ptr_controller)));
@@ -43,7 +44,12 @@ impl Collection<Ruby> for VMCollection {
                             mmtk(),
                             GCThreadTLS::to_vwt(gc_thread_tls),
                             &mut controller,
-                        )
+                        );
+
+                        // Currently the MMTk controller thread should run forever.
+                        // This is an unlikely event, but we log it anyway.
+                        warn!("The MMTk Controller Thread is quitting!");
+                        crate::unregister_gc_thread(thread::current().id());
                     })
                     .unwrap();
             }
@@ -52,6 +58,7 @@ impl Collection<Ruby> for VMCollection {
                     .name("MMTk Worker Thread".to_string())
                     .spawn(move || {
                         debug!("Hello! This is MMTk Worker Thread running!");
+                        crate::register_gc_thread(thread::current().id());
                         let ptr_worker = &mut *worker as *mut GCWorker<Ruby>;
                         let gc_thread_tls =
                             Box::into_raw(Box::new(GCThreadTLS::for_worker(ptr_worker)));
@@ -60,7 +67,12 @@ impl Collection<Ruby> for VMCollection {
                             mmtk(),
                             GCThreadTLS::to_vwt(gc_thread_tls),
                             &mut worker,
-                        )
+                        );
+
+                        // Currently all MMTk worker threads should run forever.
+                        // This is an unlikely event, but we log it anyway.
+                        warn!("An MMTk Worker Thread is quitting!");
+                        crate::unregister_gc_thread(thread::current().id());
                     })
                     .unwrap();
             }

mmtk/src/lib.rs

@@ -3,6 +3,11 @@ extern crate mmtk;
 #[macro_use]
 extern crate log;
 
+use std::collections::HashSet;
+use std::panic::PanicInfo;
+use std::sync::Mutex;
+use std::thread::ThreadId;
+
 use abi::RubyUpcalls;
 use binding::{RubyBinding, RubyBindingFast};
 use mmtk::vm::edge_shape::{SimpleEdge, UnimplementedMemorySlice};
@@ -65,3 +70,60 @@ pub fn mmtk() -> &'static MMTK<Ruby> {
 pub fn upcalls() -> &'static RubyUpcalls {
     binding().upcalls()
 }
+
+pub static GC_THREADS: OnceCell<Mutex<HashSet<ThreadId>>> = OnceCell::new();
+
+pub(crate) fn register_gc_thread(thread_id: ThreadId) {
+    let mut gc_threads = GC_THREADS.get().unwrap().lock().unwrap();
+    gc_threads.insert(thread_id);
+}
+
+pub(crate) fn unregister_gc_thread(thread_id: ThreadId) {
+    let mut gc_threads = GC_THREADS.get().unwrap().lock().unwrap();
+    gc_threads.remove(&thread_id);
+}
+
+pub(crate) fn is_gc_thread(thread_id: ThreadId) -> bool {
+    let gc_threads = GC_THREADS.get().unwrap().lock().unwrap();
+    gc_threads.contains(&thread_id)
+}
+
+fn handle_gc_thread_panic(panic_info: &PanicInfo) {
+    eprintln!("ERROR: An MMTk GC thread panicked.  This is a bug.");
+    eprintln!("{panic_info}");
+
+    let bt = std::backtrace::Backtrace::capture();
+    match bt.status() {
+        std::backtrace::BacktraceStatus::Unsupported => {
+            eprintln!("Backtrace is unsupported.")
+        }
+        std::backtrace::BacktraceStatus::Disabled => {
+            eprintln!("Backtrace is disabled.");
+            eprintln!("run with `RUST_BACKTRACE=1` environment variable to display a backtrace");
+        }
+        std::backtrace::BacktraceStatus::Captured => {
+            eprintln!("{bt}");
+        }
+        s => {
+            eprintln!("Unknown backtrace status: {s:?}");
+        }
+    }
+
+    std::process::abort();
+}
+
+pub(crate) fn set_panic_hook() {
+    if GC_THREADS.set(Default::default()).is_err() {
+        return;
+    }
+
+    let old_hook = std::panic::take_hook();
+
+    std::panic::set_hook(Box::new(move |panic_info| {
+        if is_gc_thread(std::thread::current().id()) {
+            handle_gc_thread_panic(panic_info);
+        } else {
+            old_hook(panic_info);
+        }
+    }));
+}