MallocMS page accounting

.github/scripts/ci-doc.sh

@@ -20,6 +20,7 @@ if ! cat $project_root/src/plan/mod.rs | grep -q "pub mod mygc;"; then
 fi
 cargo build
 
-cargo install mdbook
+# Install mdbook using the stable toolchain (mdbook uses scoped-tls which requires rust 1.59.0)
+cargo +stable install mdbook
 mdbook build $project_root/docs/portingguide
 mdbook build $project_root/docs/tutorial

.github/workflows/api-check.yml

@@ -28,4 +28,4 @@ jobs:
       - name: Install cargo-public-api
         run: cargo install cargo-public-api
       - name: API Diff
-        run: cargo public-api --diff-git-checkouts ${GITHUB_BASE_REF} ${{ github.event.pull_request.head.sha }} --deny=all
+        run: cargo public-api --diff-git-checkouts origin/${GITHUB_BASE_REF} ${{ github.event.pull_request.head.sha }} --deny=all

src/policy/mallocspace/metadata.rs

@@ -48,7 +48,7 @@ pub(crate) const ACTIVE_CHUNK_METADATA_SPEC: SideMetadataSpec =
 // XXX: This metadata spec is currently unused as we need to add a performant way to calculate
 // how many pages are active in this metadata spec. Explore SIMD vectorization with 8-bit integers
 pub(crate) const ACTIVE_PAGE_METADATA_SPEC: SideMetadataSpec =
-    crate::util::metadata::side_metadata::spec_defs::MS_ACTIVE_PAGE;
+    crate::util::metadata::side_metadata::spec_defs::MALLOC_MS_ACTIVE_PAGE;
 
 pub(crate) const OFFSET_MALLOC_METADATA_SPEC: SideMetadataSpec =
     crate::util::metadata::side_metadata::spec_defs::MS_OFFSET_MALLOC;
@@ -173,6 +173,15 @@ pub unsafe fn is_marked_unsafe<VM: VMBinding>(object: ObjectReference) -> bool {
     VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.load::<VM, u8>(object, None) == 1
 }
 
+/// Set the page mark from 0 to 1. Return true if we set it successfully in this call.
+pub(super) fn compare_exchange_set_page_mark(page_addr: Address) -> bool {
+    // The spec has 1 byte per each page. So it won't be the case that other threads may race and access other bits for the spec.
+    // If the compare-exchange fails, we know the byte was set to 1 before this call.
+    ACTIVE_PAGE_METADATA_SPEC
+        .compare_exchange_atomic::<u8>(page_addr, 0, 1, Ordering::SeqCst, Ordering::SeqCst)
+        .is_ok()
+}
+
 #[allow(unused)]
 pub(super) fn is_page_marked(page_addr: Address) -> bool {
     ACTIVE_PAGE_METADATA_SPEC.load_atomic::<u8>(page_addr, Ordering::SeqCst) == 1
@@ -217,6 +226,7 @@ pub fn unset_alloc_bit(object: ObjectReference) {
     alloc_bit::unset_alloc_bit(object);
 }
 
+#[allow(unused)]
 pub(super) fn set_page_mark(page_addr: Address) {
     ACTIVE_PAGE_METADATA_SPEC.store_atomic::<u8>(page_addr, 1, Ordering::SeqCst);
 }
@@ -246,10 +256,15 @@ pub unsafe fn unset_mark_bit<VM: VMBinding>(object: ObjectReference) {
     VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.store::<VM, u8>(object, 0, None);
 }
 
+#[allow(unused)]
 pub(super) unsafe fn unset_page_mark_unsafe(page_addr: Address) {
     ACTIVE_PAGE_METADATA_SPEC.store::<u8>(page_addr, 0)
 }
 
+pub(super) unsafe fn bulk_zero_page_mark(page_addr: Address, size: usize) {
+    ACTIVE_PAGE_METADATA_SPEC.bzero_metadata(page_addr, size)
+}
+
 pub(super) unsafe fn unset_chunk_mark_unsafe(chunk_start: Address) {
     ACTIVE_CHUNK_METADATA_SPEC.store::<u8>(chunk_start, 0)
 }

src/util/alloc/allocator.rs

@@ -250,6 +250,7 @@ pub trait Allocator<VM: VMBinding>: Downcast {
                             _allocation_bytes,
                             *plan.options.analysis_factor
                         );
+
                         plan.analysis_manager.alloc_hook(size, align, offset);
                     }
                 }

src/util/malloc/library.rs

@@ -13,12 +13,19 @@ pub use self::libc_malloc::*;
 #[cfg(feature = "malloc_mimalloc")]
 pub use self::mimalloc::*;
 
+/// When we count page usage of library malloc, we assume they allocate in pages. For some malloc implementations,
+/// they may use a larger page (e.g. mimalloc's 64K page). For libraries that we are not sure, we assume they use
+/// normal 4k pages.
+pub const BYTES_IN_MALLOC_PAGE: usize = 1 << LOG_BYTES_IN_MALLOC_PAGE;
+
 // Different malloc libraries
 
 // TODO: We should conditinally include some methods in the module, such as posix extension and GNU extension.
 
 #[cfg(feature = "malloc_jemalloc")]
 mod jemalloc {
+    // Normal 4K page
+    pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = crate::util::constants::LOG_BYTES_IN_PAGE;
     // ANSI C
     pub use jemalloc_sys::{calloc, free, malloc, realloc};
     // Posix
@@ -29,6 +36,8 @@ mod jemalloc {
 
 #[cfg(feature = "malloc_mimalloc")]
 mod mimalloc {
+    // MiMalloc 64K Page
+    pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = 16;
     // ANSI C
     pub use mimalloc_sys::{
         mi_calloc as calloc, mi_free as free, mi_malloc as malloc, mi_realloc as realloc,
@@ -41,6 +50,8 @@ mod mimalloc {
 
 #[cfg(feature = "malloc_hoard")]
 mod hoard {
+    // Normal 4K page
+    pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = crate::util::constants::LOG_BYTES_IN_PAGE;
     // ANSI C
     pub use hoard_sys::{calloc, free, malloc, realloc};
     // Posix
@@ -56,6 +67,8 @@ mod hoard {
     feature = "malloc_hoard",
 )))]
 mod libc_malloc {
+    // Normal 4K page
+    pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = crate::util::constants::LOG_BYTES_IN_PAGE;
     // ANSI C
     pub use libc::{calloc, free, malloc, realloc};
     // Posix

src/util/metadata/side_metadata/spec_defs.rs

@@ -66,7 +66,7 @@ define_side_metadata_specs!(
 define_side_metadata_specs!(
     last_spec_as LAST_LOCAL_SIDE_METADATA_SPEC,
     // Mark pages by (malloc) marksweep
-    MS_ACTIVE_PAGE  = (global: false, log_num_of_bits: 3, log_bytes_in_region: LOG_BYTES_IN_PAGE as usize),
+    MALLOC_MS_ACTIVE_PAGE  = (global: false, log_num_of_bits: 3, log_bytes_in_region: crate::util::malloc::library::LOG_BYTES_IN_MALLOC_PAGE as usize),
     // Record objects allocated with some offset
     MS_OFFSET_MALLOC = (global: false, log_num_of_bits: 0, log_bytes_in_region: LOG_MIN_OBJECT_SIZE as usize),
     // Mark lines by immix