diff --git a/.github/scripts/ci-test-stdlib.sh b/.github/scripts/ci-test-stdlib.sh index f2858768..f50d9b73 100755 --- a/.github/scripts/ci-test-stdlib.sh +++ b/.github/scripts/ci-test-stdlib.sh @@ -22,6 +22,8 @@ declare -a tests_to_skip=( "SparseArrays" # Running LinearAlgebra in a separate job "LinearAlgebra" + # Skipping Distributed tests + "Distributed" ) # These tests need multiple workers. declare -a tests_with_multi_workers=( diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72d7a7ac..1e45266e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: fail-fast: false matrix: gc_plan: [Immix] - moving: [Non_Moving] + moving: [Non_Moving, Moving] uses: ./.github/workflows/binding-tests.yml with: gc_plan: ${{ matrix.gc_plan }} diff --git a/.gitignore b/.gitignore index 3b9f4dee..6707474f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea/ julia/*.o julia/*.dbj.obj -.vscode \ No newline at end of file +.vscode +mmtk/src/julia_types.rs diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index a4d83a52..65fc59ea 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -995,18 +995,18 @@ checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index a1c42705..6a04bbf8 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,15 +40,17 @@ memoffset = "*" # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" } [features] -default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning"] +# We must build with default features +default = ["mmtk/vm_space", "julia_copy_stack", "mmtk/object_pinning", "mmtk/is_mmtk_object", "mmtk/vo_bit_access"] -# Plans +# Default features +julia_copy_stack = [] + +# Plans: choose one nogc = [] immix = [] stickyimmix = ["mmtk/sticky_immix_non_moving_nursery", "mmtk/immix_smaller_block"] marksweep = [] -object_pinning = ["mmtk/object_pinning"] # This feature disables moving non_moving = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"] -julia_copy_stack = [] diff --git a/mmtk/api/mmtk.h b/mmtk/api/mmtk.h index 8e52239a..06561871 100644 --- a/mmtk/api/mmtk.h +++ b/mmtk/api/mmtk.h @@ -49,7 +49,9 @@ extern int mmtk_object_is_managed_by_mmtk(void* addr); extern void mmtk_runtime_panic(void); extern void mmtk_unreachable(void); extern unsigned char mmtk_pin_object(void* obj); -extern bool mmtk_is_pinned(void* obj); +extern bool mmtk_is_object_pinned(void* obj); +extern unsigned char mmtk_pin_pointer(void* ptr); +extern bool mmtk_is_pointer_pinned(void* ptr); extern const char* get_mmtk_version(void); extern void mmtk_set_vm_space(void* addr, size_t size); @@ -85,7 +87,7 @@ extern void mmtk_block_thread_for_gc(void); extern void* mmtk_new_mutator_iterator(void); extern void* mmtk_get_next_mutator_tls(void*); extern void* mmtk_close_mutator_iterator(void*); - +extern void mmtk_store_obj_size_c(void* obj, size_t size); /** * VM Accounting @@ -106,6 +108,11 @@ extern void mmtk_add_phantom_candidate(void* ref); extern void mmtk_harness_begin(void *tls); extern void mmtk_harness_end(void); +/** + * Copying + */ +extern const void* MMTK_SIDE_VO_BIT_BASE_ADDRESS; + #ifdef __cplusplus } #endif diff --git a/mmtk/build.rs b/mmtk/build.rs index 3871e374..0d386983 100644 --- a/mmtk/build.rs +++ b/mmtk/build.rs @@ -54,6 +54,7 @@ fn main() { let bindings = bindgen::Builder::default() .header(format!("{}/src/julia.h", julia_dir)) .header(format!("{}/src/julia_internal.h", julia_dir)) + .header(format!("{}/src/gc-common.h", julia_dir)) // Including the paths to depending .h files .clang_arg("-I") .clang_arg(format!("{}/mmtk/api", mmtk_dir)) @@ -77,6 +78,7 @@ fn main() { .allowlist_item("jl_bt_element_t") .allowlist_item("jl_taggedvalue_t") .allowlist_item("MMTkMutatorContext") + .allowlist_item("_bigval_t") // --opaque-type MMTkMutatorContext .opaque_type("MMTkMutatorContext") // compile using c++ diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index f58bb500..08f0dafc 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -350,6 +350,7 @@ pub extern "C" fn mmtk_set_vm_space(start: Address, size: usize) { #[cfg(feature = "stickyimmix")] set_side_log_bit_for_region(start, size); + set_side_vo_bit_for_region(start, size); } #[no_mangle] @@ -381,6 +382,8 @@ pub extern "C" fn mmtk_memory_region_copy( pub extern "C" fn mmtk_immortal_region_post_alloc(start: Address, size: usize) { #[cfg(feature = "stickyimmix")] set_side_log_bit_for_region(start, size); + + set_side_vo_bit_for_region(start, size); } #[cfg(feature = "stickyimmix")] @@ -393,6 +396,19 @@ fn set_side_log_bit_for_region(start: Address, size: usize) { } } +// We have to set VO bit even if this is a non_moving build. Otherwise, assertions in mmtk-core +// will complain about seeing objects without VO bit. +fn set_side_vo_bit_for_region(start: Address, size: usize) { + debug!( + "Bulk set VO bit {} to {} ({} bytes)", + start, + start + size, + size + ); + + mmtk::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_SPEC.bset_metadata(start, size); +} + #[no_mangle] pub extern "C" fn mmtk_object_reference_write_post( mutator: *mut Mutator, @@ -427,6 +443,11 @@ pub extern "C" fn mmtk_object_reference_write_slow( pub static MMTK_SIDE_LOG_BIT_BASE_ADDRESS: Address = mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; +/// VO bit base address +#[no_mangle] +pub static MMTK_SIDE_VO_BIT_BASE_ADDRESS: Address = + mmtk::util::metadata::side_metadata::VO_BIT_SIDE_METADATA_ADDR; + #[no_mangle] pub extern "C" fn mmtk_object_is_managed_by_mmtk(addr: usize) -> bool { crate::api::mmtk_is_mapped_address(unsafe { Address::from_usize(addr) }) @@ -464,57 +485,133 @@ pub extern "C" fn mmtk_get_obj_size(obj: ObjectReference) -> usize { } } -#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))] +#[allow(unused_variables)] +fn assert_is_object(object: ObjectReference) { + // The checks are quite expensive. Dont run it in normal builds. + const ASSERT_OBJECT: bool = false; + if ASSERT_OBJECT { + #[cfg(debug_assertions)] + { + use crate::object_model::{is_object_in_immixspace, is_object_in_los}; + if !mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { + panic!("{} is not managed by MMTk", object); + } + if !is_object_in_immixspace(&object) && !is_object_in_los(&object) { + // We will use VO bit in the following check. But if the object is not in immix space or LOS, we cannot do the check. + return; + } + if !object + .to_raw_address() + .is_aligned_to(ObjectReference::ALIGNMENT) + { + panic!( + "{} is not aligned, it cannot be an object reference", + object + ) + } + if memory_manager::is_mmtk_object(object.to_raw_address()).is_none() { + error!("{} is not an object", object); + if let Some(base_ref) = memory_manager::find_object_from_internal_pointer( + object.to_raw_address(), + usize::MAX, + ) { + panic!("{} is an internal pointer of {}", object, base_ref); + } else { + panic!( + "{} is not recognised as an object reference, or an internal reference", + object + ); + } + } + } + } +} #[no_mangle] pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool { - // We may in the future replace this with a check for the immix space (bound check), which should be much cheaper. - if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { - memory_manager::pin_object(object) - } else { - debug!("Object is not managed by mmtk - (un)pinning it via this function isn't supported."); - false - } + assert_is_object(object); + crate::early_return_for_non_moving_build!(false); + memory_manager::pin_object(object) } -#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))] #[no_mangle] pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool { - if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { - memory_manager::unpin_object(object) - } else { - debug!("Object is not managed by mmtk - (un)pinning it via this function isn't supported."); - false - } + assert_is_object(object); + crate::early_return_for_non_moving_build!(false); + memory_manager::unpin_object(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_is_object_pinned(object: ObjectReference) -> bool { + assert_is_object(object); + crate::early_return_for_non_moving_build!(false); + + memory_manager::is_pinned(object) +} + +macro_rules! handle_potential_internal_pointer { + ($func: path, $addr: expr) => {{ + if $addr.is_aligned_to(ObjectReference::ALIGNMENT) { + if let Some(obj) = memory_manager::is_mmtk_object($addr) { + return $func(obj); + } + } + let maybe_objref = memory_manager::find_object_from_internal_pointer($addr, usize::MAX); + if let Some(obj) = maybe_objref { + trace!( + "Attempt to pin {:?}, but it is an internal reference of {:?}", + $addr, + obj + ); + $func(obj) + } else { + warn!( + "Attempt to pin {:?}, but it is not recognised as a object", + $addr + ); + false + } + }}; } -#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))] #[no_mangle] -pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool { - if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { - memory_manager::is_pinned(object) +pub extern "C" fn mmtk_pin_pointer(addr: Address) -> bool { + crate::early_return_for_non_moving_build!(false); + + if crate::object_model::is_addr_in_immixspace(addr) { + handle_potential_internal_pointer!(memory_manager::pin_object, addr) } else { - debug!("Object is not managed by mmtk - checking via this function isn't supported."); + debug!("Object is not in Immix space. MMTk will not move the object. No need to pin the object."); false } } -// If the `non-moving` feature is selected, pinning/unpinning is a noop and simply returns false -#[cfg(all(feature = "object_pinning", feature = "non_moving"))] #[no_mangle] -pub extern "C" fn mmtk_pin_object(_object: ObjectReference) -> bool { - false -} +pub extern "C" fn mmtk_unpin_pointer(addr: Address) -> bool { + crate::early_return_for_non_moving_build!(false); -#[cfg(all(feature = "object_pinning", feature = "non_moving"))] -#[no_mangle] -pub extern "C" fn mmtk_unpin_object(_object: ObjectReference) -> bool { - false + if crate::object_model::is_addr_in_immixspace(addr) { + handle_potential_internal_pointer!(memory_manager::unpin_object, addr) + } else { + debug!("Object is not in Immix space. MMTk will not move the object. No need to unpin the object."); + false + } } -#[cfg(all(feature = "object_pinning", feature = "non_moving"))] #[no_mangle] -pub extern "C" fn mmtk_is_pinned(_object: ObjectReference) -> bool { - false +pub extern "C" fn mmtk_is_pointer_pinned(addr: Address) -> bool { + crate::early_return_for_non_moving_build!(false); + + if crate::object_model::is_addr_in_immixspace(addr) { + handle_potential_internal_pointer!(memory_manager::is_pinned, addr) + } else if !mmtk_object_is_managed_by_mmtk(addr.as_usize()) { + debug!( + "Object is not in Immix space. MMTk will not move the object. We assume it is pinned." + ); + true + } else { + debug!("Object is not managed by mmtk - checking pinning state via this function isn't supported."); + false + } } #[no_mangle] diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 2ba88641..486c7886 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -4,7 +4,7 @@ use crate::{ jl_hrtime, jl_throw_out_of_memory_error, }; use crate::{JuliaVM, USER_TRIGGERED_GC}; -use log::{info, trace}; +use log::{debug, trace}; use mmtk::util::alloc::AllocationError; use mmtk::util::heap::GCTriggerPolicy; use mmtk::util::opaque_pointer::*; @@ -15,6 +15,7 @@ use std::sync::atomic::{AtomicBool, AtomicIsize, AtomicU64, Ordering}; use crate::{BLOCK_FOR_GC, STW_COND, WORLD_HAS_STOPPED}; pub static GC_START: AtomicU64 = AtomicU64::new(0); +static CURRENT_GC_MAY_MOVE: AtomicBool = AtomicBool::new(true); use std::collections::HashSet; use std::sync::RwLock; @@ -52,11 +53,18 @@ impl Collection for VMCollection { trace!("Stopped the world!"); + // Store if the current GC may move objects -- we will use it when the current GC finishes. + // We cache the value here just in case MMTk may clear it before we use the value. + CURRENT_GC_MAY_MOVE.store( + crate::SINGLETON.get_plan().current_gc_may_move_object(), + Ordering::SeqCst, + ); + // Tell MMTk the stacks are ready. { use mmtk::vm::ActivePlan; for mutator in crate::active_plan::VMActivePlan::mutators() { - info!("stop_all_mutators: visiting {:?}", mutator.mutator_tls); + debug!("stop_all_mutators: visiting {:?}", mutator.mutator_tls); mutator_visitor(mutator); } } @@ -68,6 +76,9 @@ impl Collection for VMCollection { } fn resume_mutators(_tls: VMWorkerThread) { + // unpin conservative roots + crate::conservative::unpin_conservative_roots(); + // Get the end time of the GC let end = unsafe { jl_hrtime() }; trace!("gc_end = {}", end); @@ -86,7 +97,7 @@ impl Collection for VMCollection { let (_, cvar) = &*STW_COND.clone(); cvar.notify_all(); - info!( + debug!( "Live bytes = {}, total bytes = {}", crate::api::mmtk_used_bytes(), crate::api::mmtk_total_bytes() @@ -96,11 +107,11 @@ impl Collection for VMCollection { } fn block_for_gc(_tls: VMMutatorThread) { - info!("Triggered GC!"); + debug!("Triggered GC!"); unsafe { jl_gc_prepare_to_collect() }; - info!("Finished blocking mutator for GC!"); + debug!("Finished blocking mutator for GC!"); } fn spawn_gc_thread(_tls: VMThread, ctx: GCThreadContext) { @@ -158,6 +169,10 @@ pub fn is_current_gc_nursery() -> bool { } } +pub fn is_current_gc_moving() -> bool { + CURRENT_GC_MAY_MOVE.load(Ordering::SeqCst) +} + #[no_mangle] pub extern "C" fn mmtk_block_thread_for_gc() { AtomicBool::store(&BLOCK_FOR_GC, true, Ordering::SeqCst); @@ -165,7 +180,7 @@ pub extern "C" fn mmtk_block_thread_for_gc() { let (lock, cvar) = &*STW_COND.clone(); let mut count = lock.lock().unwrap(); - info!("Blocking for GC!"); + debug!("Blocking for GC!"); AtomicBool::store(&WORLD_HAS_STOPPED, true, Ordering::SeqCst); diff --git a/mmtk/src/conservative.rs b/mmtk/src/conservative.rs new file mode 100644 index 00000000..0ce6c80e --- /dev/null +++ b/mmtk/src/conservative.rs @@ -0,0 +1,125 @@ +use crate::julia_types::*; +use mmtk::memory_manager; +use mmtk::util::constants::BYTES_IN_ADDRESS; +use mmtk::util::{Address, ObjectReference}; +use std::collections::HashSet; +use std::sync::Mutex; +lazy_static! { + pub static ref CONSERVATIVE_ROOTS: Mutex> = Mutex::new(HashSet::new()); +} +pub fn pin_conservative_roots() { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + let mut roots = CONSERVATIVE_ROOTS.lock().unwrap(); + let n_roots = roots.len(); + roots.retain(|obj| mmtk::memory_manager::pin_object(*obj)); + let n_pinned = roots.len(); + log::debug!("Conservative roots: {}, pinned: {}", n_roots, n_pinned); +} +pub fn unpin_conservative_roots() { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + let mut roots = CONSERVATIVE_ROOTS.lock().unwrap(); + let n_pinned = roots.len(); + let mut n_live = 0; + roots.drain().for_each(|obj| { + if mmtk::memory_manager::is_live_object(obj) { + n_live += 1; + mmtk::memory_manager::unpin_object(obj); + } + }); + log::debug!( + "Conservative roots: pinned: {}, unpinned/live {}", + n_pinned, + n_live + ); +} +pub fn mmtk_conservative_scan_task_stack(ta: *const jl_task_t) { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + log::debug!("mmtk_conservative_scan_native_stack begin ta = {:?}", ta); + let mut active_start = Address::ZERO; + let mut active_end = Address::ZERO; + let mut total_start = Address::ZERO; + let mut total_end = Address::ZERO; + unsafe { + crate::jl_active_task_stack( + ta, + &mut active_start as _, + &mut active_end as _, + &mut total_start as _, + &mut total_end as _, + ) + }; + log::debug!( + "mmtk_conservative_scan_native_stack continue, active = {},{}, total = {},{}", + active_start, + active_end, + total_start, + total_end, + ); + + let size = active_end - active_start; + + if !active_start.is_zero() { + log::debug!("Conservatively scan the stack"); + // See jl_guard_size + // TODO: Are we sure there are always guard pages we need to skip? + const JL_GUARD_PAGE: usize = 4096 * 8; + let guard_page_start = active_start + JL_GUARD_PAGE; + log::debug!("Skip guard page: {}, {}", active_start, guard_page_start); + conservative_scan_range(guard_page_start, active_start + size); + } else { + log::warn!("Skip stack for {:?}", ta); + } +} +pub fn mmtk_conservative_scan_task_registers(ta: *const jl_task_t) { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + let (lo, hi) = get_range(&unsafe { &*ta }.ctx); + conservative_scan_range(lo, hi); +} +pub fn mmtk_conservative_scan_ptls_registers(ptls: &mut _jl_tls_states_t) { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + let (lo, hi) = get_range(&((*ptls).gc_tls.ctx_at_the_time_gc_started)); + conservative_scan_range(lo, hi); +} +// TODO: This scans the entire context type, which is slower. +// We actually only need to scan registers. +fn get_range(ctx: &T) -> (Address, Address) { + let start = Address::from_ptr(ctx); + let ty_size = std::mem::size_of::(); + (start, start + ty_size) +} +fn conservative_scan_range(lo: Address, hi: Address) { + // The high address is exclusive + let hi = if hi.is_aligned_to(BYTES_IN_ADDRESS) { + hi - BYTES_IN_ADDRESS + } else { + hi.align_down(BYTES_IN_ADDRESS) + }; + let lo = lo.align_up(BYTES_IN_ADDRESS); + log::trace!("Scan {} (lo) {} (hi)", lo, hi); + let mut cursor = hi; + while cursor >= lo { + let addr = unsafe { cursor.load::
() }; + if let Some(obj) = is_potential_mmtk_object(addr) { + CONSERVATIVE_ROOTS.lock().unwrap().insert(obj); + } + cursor -= BYTES_IN_ADDRESS; + } +} +fn is_potential_mmtk_object(addr: Address) -> Option { + if crate::object_model::is_addr_in_immixspace(addr) { + // We only care about immix space. If the object is in other spaces, we won't move them, and we don't need to pin them. + memory_manager::find_object_from_internal_pointer(addr, usize::MAX) + } else { + None + } +} diff --git a/mmtk/src/gc_trigger.rs b/mmtk/src/gc_trigger.rs index 056892cf..a03fefc6 100644 --- a/mmtk/src/gc_trigger.rs +++ b/mmtk/src/gc_trigger.rs @@ -128,9 +128,8 @@ impl GCTriggerPolicy for JuliaGCTrigger { let heap_size = conversions::pages_to_bytes(reserved_pages_now); let user_max = self.max_total_memory.load(Ordering::Relaxed) * 80 / 100; - let alloc_diff = self.before_free_heap_size.load(Ordering::Relaxed) - - self.old_heap_size.load(Ordering::Relaxed); - let freed_diff = self.before_free_heap_size.load(Ordering::Relaxed) - heap_size; + let alloc_diff = self.before_free_heap_size.load(Ordering::Relaxed).saturating_sub(self.old_heap_size.load(Ordering::Relaxed)); + let freed_diff = self.before_free_heap_size.load(Ordering::Relaxed).saturating_sub(heap_size); self.old_heap_size.store(heap_size, Ordering::Relaxed); // update the heap target only if the user did not force a GC diff --git a/mmtk/src/julia_scanning.rs b/mmtk/src/julia_scanning.rs index ca9a6def..2a321b53 100644 --- a/mmtk/src/julia_scanning.rs +++ b/mmtk/src/julia_scanning.rs @@ -132,6 +132,12 @@ pub unsafe fn scan_julia_object>(obj: Address, clos } process_slot(closure, Address::from_ptr(parent_slot)); + let usings_backeges_slot = ::std::ptr::addr_of!((*m).usings_backedges); + if PRINT_OBJ_TYPE { + println!(" - scan parent: {:?}\n", usings_backeges_slot); + } + process_slot(closure, Address::from_ptr(usings_backeges_slot)); + // m.usings.items may be inlined in the module when the array list size <= AL_N_INLINE (cf. arraylist_new) // In that case it may be an mmtk object and not a malloced address. // If it is an mmtk object, (*m).usings.items will then be an internal pointer to the module @@ -162,7 +168,7 @@ pub unsafe fn scan_julia_object>(obj: Address, clos let ta = obj.to_ptr::(); - mmtk_scan_gcstack(ta, closure); + mmtk_scan_gcstack(ta, closure, None); let layout = (*jl_task_type).layout; debug_assert!((*layout).fielddesc_type_custom() == 0); @@ -373,10 +379,68 @@ unsafe fn mmtk_jl_genericmemory_data_owner_field_address(m: *const jl_genericmem // mmtk_jl_genericmemory_data_owner_field_address(m).load::<*const mmtk_jl_value_t>() // } -pub unsafe fn mmtk_scan_gcstack>( +pub unsafe fn mmtk_scan_gcpreserve_stack<'a, EV: SlotVisitor>( ta: *const jl_task_t, - closure: &mut EV, + closure: &'a mut EV, +) { + // process transitively pinning stack + let mut s = (*ta).gcpreserve_stack; + let (offset, lb, ub) = (0 as isize, 0 as u64, u64::MAX); + + if s != std::ptr::null_mut() { + let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); + let mut nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); + debug_assert!(nroots.as_usize() <= u32::MAX as usize); + let mut nr = nroots >> 3; + + loop { + let rts = Address::from_mut_ptr(s).shift::
(2); + let mut i = 0; + + while i < nr { + let real_addr = get_stack_addr(rts.shift::
(i as isize), offset, lb, ub); + + let slot = read_stack(rts.shift::
(i as isize), offset, lb, ub); + use crate::julia_finalizer::gc_ptr_tag; + // malloced pointer tagged in jl_gc_add_quiescent + // skip both the next element (native function), and the object + if slot & 3usize == 3 { + i += 2; + continue; + } + + // pointer is not malloced but function is native, so skip it + if gc_ptr_tag(slot, 1) { + i += 2; + continue; + } + + process_slot(closure, real_addr); + i += 1; + } + + let s_prev_address = ::std::ptr::addr_of!((*s).prev); + let sprev = read_stack(Address::from_ptr(s_prev_address), offset, lb, ub); + if sprev.is_zero() { + break; + } + + s = sprev.to_mut_ptr::(); + let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); + let new_nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); + nroots = new_nroots; + nr = nroots >> 3; + continue; + } + } +} + +pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor>( + ta: *const jl_task_t, + mut closure: &'a mut EV, + mut pclosure: Option<&'a mut EV>, ) { + // process Julia's standard shadow (GC) stack let stkbuf = (*ta).ctx.stkbuf; let copy_stack = (*ta).ctx.copy_stack_custom(); @@ -404,16 +468,28 @@ pub unsafe fn mmtk_scan_gcstack>( let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); let mut nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); debug_assert!(nroots.as_usize() as u32 <= u32::MAX); - let mut nr = nroots >> 2; + let mut nr = nroots >> 3; loop { + // if the 'pin' bit on the root type is not set, must transitively pin + // and therefore use transitive pinning closure + let closure_to_use: &mut &mut EV = if (nroots.as_usize() & 4) == 0 { + &mut closure + } else { + // otherwise, use the pinning closure (if available) + match &mut pclosure { + Some(c) => c, + None => &mut closure, + } + }; + let rts = Address::from_mut_ptr(s).shift::
(2); let mut i = 0; while i < nr { if (nroots.as_usize() & 1) != 0 { let slot = read_stack(rts.shift::
(i as isize), offset, lb, ub); let real_addr = get_stack_addr(slot, offset, lb, ub); - process_slot(closure, real_addr); + process_slot(*closure_to_use, real_addr); } else { let real_addr = get_stack_addr(rts.shift::
(i as isize), offset, lb, ub); @@ -429,12 +505,12 @@ pub unsafe fn mmtk_scan_gcstack>( // pointer is not malloced but function is native, so skip it if gc_ptr_tag(slot, 1) { - process_offset_slot(closure, real_addr, 1); + process_offset_slot(*closure_to_use, real_addr, 1); i += 2; continue; } - process_slot(closure, real_addr); + process_slot(*closure_to_use, real_addr); } i += 1; @@ -450,7 +526,7 @@ pub unsafe fn mmtk_scan_gcstack>( let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); let new_nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); nroots = new_nroots; - nr = nroots >> 2; + nr = nroots >> 3; continue; } } diff --git a/mmtk/src/julia_types.rs b/mmtk/src/julia_types.rs index 530c2d96..f4f15f4e 100644 --- a/mmtk/src/julia_types.rs +++ b/mmtk/src/julia_types.rs @@ -174,6 +174,9 @@ pub struct std_atomic<_Tp> { pub _M_i: _Tp, } pub type std_atomic_value_type<_Tp> = _Tp; +pub type __uint16_t = ::std::os::raw::c_ushort; +pub type __uint32_t = ::std::os::raw::c_uint; +pub type __uint64_t = ::std::os::raw::c_ulong; pub type __sig_atomic_t = ::std::os::raw::c_int; pub type jl_gcframe_t = _jl_gcframe_t; #[repr(C)] @@ -187,6 +190,7 @@ const _: () = { ["Alignment of __sigset_t"][::std::mem::align_of::<__sigset_t>() - 8usize]; ["Offset of field: __sigset_t::__val"][::std::mem::offset_of!(__sigset_t, __val) - 0usize]; }; +pub type sigset_t = __sigset_t; pub type pthread_t = ::std::os::raw::c_ulong; #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -254,6 +258,133 @@ pub struct _jl_value_t { _unused: [u8; 0], } pub type sig_atomic_t = __sig_atomic_t; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct stack_t { + pub ss_sp: *mut ::std::os::raw::c_void, + pub ss_flags: ::std::os::raw::c_int, + pub ss_size: usize, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of stack_t"][::std::mem::size_of::() - 24usize]; + ["Alignment of stack_t"][::std::mem::align_of::() - 8usize]; + ["Offset of field: stack_t::ss_sp"][::std::mem::offset_of!(stack_t, ss_sp) - 0usize]; + ["Offset of field: stack_t::ss_flags"][::std::mem::offset_of!(stack_t, ss_flags) - 8usize]; + ["Offset of field: stack_t::ss_size"][::std::mem::offset_of!(stack_t, ss_size) - 16usize]; +}; +pub type greg_t = ::std::os::raw::c_longlong; +pub type gregset_t = [greg_t; 23usize]; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _libc_fpxreg { + pub significand: [::std::os::raw::c_ushort; 4usize], + pub exponent: ::std::os::raw::c_ushort, + pub __glibc_reserved1: [::std::os::raw::c_ushort; 3usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _libc_fpxreg"][::std::mem::size_of::<_libc_fpxreg>() - 16usize]; + ["Alignment of _libc_fpxreg"][::std::mem::align_of::<_libc_fpxreg>() - 2usize]; + ["Offset of field: _libc_fpxreg::significand"] + [::std::mem::offset_of!(_libc_fpxreg, significand) - 0usize]; + ["Offset of field: _libc_fpxreg::exponent"] + [::std::mem::offset_of!(_libc_fpxreg, exponent) - 8usize]; + ["Offset of field: _libc_fpxreg::__glibc_reserved1"] + [::std::mem::offset_of!(_libc_fpxreg, __glibc_reserved1) - 10usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _libc_xmmreg { + pub element: [__uint32_t; 4usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _libc_xmmreg"][::std::mem::size_of::<_libc_xmmreg>() - 16usize]; + ["Alignment of _libc_xmmreg"][::std::mem::align_of::<_libc_xmmreg>() - 4usize]; + ["Offset of field: _libc_xmmreg::element"] + [::std::mem::offset_of!(_libc_xmmreg, element) - 0usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _libc_fpstate { + pub cwd: __uint16_t, + pub swd: __uint16_t, + pub ftw: __uint16_t, + pub fop: __uint16_t, + pub rip: __uint64_t, + pub rdp: __uint64_t, + pub mxcsr: __uint32_t, + pub mxcr_mask: __uint32_t, + pub _st: [_libc_fpxreg; 8usize], + pub _xmm: [_libc_xmmreg; 16usize], + pub __glibc_reserved1: [__uint32_t; 24usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _libc_fpstate"][::std::mem::size_of::<_libc_fpstate>() - 512usize]; + ["Alignment of _libc_fpstate"][::std::mem::align_of::<_libc_fpstate>() - 8usize]; + ["Offset of field: _libc_fpstate::cwd"][::std::mem::offset_of!(_libc_fpstate, cwd) - 0usize]; + ["Offset of field: _libc_fpstate::swd"][::std::mem::offset_of!(_libc_fpstate, swd) - 2usize]; + ["Offset of field: _libc_fpstate::ftw"][::std::mem::offset_of!(_libc_fpstate, ftw) - 4usize]; + ["Offset of field: _libc_fpstate::fop"][::std::mem::offset_of!(_libc_fpstate, fop) - 6usize]; + ["Offset of field: _libc_fpstate::rip"][::std::mem::offset_of!(_libc_fpstate, rip) - 8usize]; + ["Offset of field: _libc_fpstate::rdp"][::std::mem::offset_of!(_libc_fpstate, rdp) - 16usize]; + ["Offset of field: _libc_fpstate::mxcsr"] + [::std::mem::offset_of!(_libc_fpstate, mxcsr) - 24usize]; + ["Offset of field: _libc_fpstate::mxcr_mask"] + [::std::mem::offset_of!(_libc_fpstate, mxcr_mask) - 28usize]; + ["Offset of field: _libc_fpstate::_st"][::std::mem::offset_of!(_libc_fpstate, _st) - 32usize]; + ["Offset of field: _libc_fpstate::_xmm"] + [::std::mem::offset_of!(_libc_fpstate, _xmm) - 160usize]; + ["Offset of field: _libc_fpstate::__glibc_reserved1"] + [::std::mem::offset_of!(_libc_fpstate, __glibc_reserved1) - 416usize]; +}; +pub type fpregset_t = *mut _libc_fpstate; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct mcontext_t { + pub gregs: gregset_t, + pub fpregs: fpregset_t, + pub __reserved1: [::std::os::raw::c_ulonglong; 8usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of mcontext_t"][::std::mem::size_of::() - 256usize]; + ["Alignment of mcontext_t"][::std::mem::align_of::() - 8usize]; + ["Offset of field: mcontext_t::gregs"][::std::mem::offset_of!(mcontext_t, gregs) - 0usize]; + ["Offset of field: mcontext_t::fpregs"][::std::mem::offset_of!(mcontext_t, fpregs) - 184usize]; + ["Offset of field: mcontext_t::__reserved1"] + [::std::mem::offset_of!(mcontext_t, __reserved1) - 192usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ucontext_t { + pub uc_flags: ::std::os::raw::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: stack_t, + pub uc_mcontext: mcontext_t, + pub uc_sigmask: sigset_t, + pub __fpregs_mem: _libc_fpstate, + pub __ssp: [::std::os::raw::c_ulonglong; 4usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of ucontext_t"][::std::mem::size_of::() - 968usize]; + ["Alignment of ucontext_t"][::std::mem::align_of::() - 8usize]; + ["Offset of field: ucontext_t::uc_flags"] + [::std::mem::offset_of!(ucontext_t, uc_flags) - 0usize]; + ["Offset of field: ucontext_t::uc_link"][::std::mem::offset_of!(ucontext_t, uc_link) - 8usize]; + ["Offset of field: ucontext_t::uc_stack"] + [::std::mem::offset_of!(ucontext_t, uc_stack) - 16usize]; + ["Offset of field: ucontext_t::uc_mcontext"] + [::std::mem::offset_of!(ucontext_t, uc_mcontext) - 40usize]; + ["Offset of field: ucontext_t::uc_sigmask"] + [::std::mem::offset_of!(ucontext_t, uc_sigmask) - 296usize]; + ["Offset of field: ucontext_t::__fpregs_mem"] + [::std::mem::offset_of!(ucontext_t, __fpregs_mem) - 424usize]; + ["Offset of field: ucontext_t::__ssp"][::std::mem::offset_of!(ucontext_t, __ssp) - 936usize]; +}; pub type jl_value_t = _jl_value_t; #[repr(C)] #[repr(align(8))] @@ -269,15 +400,18 @@ const _: () = { pub struct jl_gc_tls_states_t { pub mmtk_mutator: MMTkMutatorContext, pub malloc_sz_since_last_poll: std_atomic, + pub ctx_at_the_time_gc_started: ucontext_t, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of jl_gc_tls_states_t"][::std::mem::size_of::() - 704usize]; + ["Size of jl_gc_tls_states_t"][::std::mem::size_of::() - 1672usize]; ["Alignment of jl_gc_tls_states_t"][::std::mem::align_of::() - 8usize]; ["Offset of field: jl_gc_tls_states_t::mmtk_mutator"] [::std::mem::offset_of!(jl_gc_tls_states_t, mmtk_mutator) - 0usize]; ["Offset of field: jl_gc_tls_states_t::malloc_sz_since_last_poll"] [::std::mem::offset_of!(jl_gc_tls_states_t, malloc_sz_since_last_poll) - 696usize]; + ["Offset of field: jl_gc_tls_states_t::ctx_at_the_time_gc_started"] + [::std::mem::offset_of!(jl_gc_tls_states_t, ctx_at_the_time_gc_started) - 704usize]; }; #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -541,7 +675,7 @@ pub struct _jl_tls_states_t { } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of _jl_tls_states_t"][::std::mem::size_of::<_jl_tls_states_t>() - 2560usize]; + ["Size of _jl_tls_states_t"][::std::mem::size_of::<_jl_tls_states_t>() - 3528usize]; ["Alignment of _jl_tls_states_t"][::std::mem::align_of::<_jl_tls_states_t>() - 8usize]; ["Offset of field: _jl_tls_states_t::tid"] [::std::mem::offset_of!(_jl_tls_states_t, tid) - 0usize]; @@ -566,53 +700,53 @@ const _: () = { ["Offset of field: _jl_tls_states_t::gc_tls"] [::std::mem::offset_of!(_jl_tls_states_t, gc_tls) - 40usize]; ["Offset of field: _jl_tls_states_t::gc_tls_common"] - [::std::mem::offset_of!(_jl_tls_states_t, gc_tls_common) - 744usize]; + [::std::mem::offset_of!(_jl_tls_states_t, gc_tls_common) - 1712usize]; ["Offset of field: _jl_tls_states_t::lazily_freed_mtarraylist_buffers"] - [::std::mem::offset_of!(_jl_tls_states_t, lazily_freed_mtarraylist_buffers) - 2024usize]; + [::std::mem::offset_of!(_jl_tls_states_t, lazily_freed_mtarraylist_buffers) - 2992usize]; ["Offset of field: _jl_tls_states_t::defer_signal"] - [::std::mem::offset_of!(_jl_tls_states_t, defer_signal) - 2088usize]; + [::std::mem::offset_of!(_jl_tls_states_t, defer_signal) - 3056usize]; ["Offset of field: _jl_tls_states_t::current_task"] - [::std::mem::offset_of!(_jl_tls_states_t, current_task) - 2096usize]; + [::std::mem::offset_of!(_jl_tls_states_t, current_task) - 3064usize]; ["Offset of field: _jl_tls_states_t::next_task"] - [::std::mem::offset_of!(_jl_tls_states_t, next_task) - 2104usize]; + [::std::mem::offset_of!(_jl_tls_states_t, next_task) - 3072usize]; ["Offset of field: _jl_tls_states_t::previous_task"] - [::std::mem::offset_of!(_jl_tls_states_t, previous_task) - 2112usize]; + [::std::mem::offset_of!(_jl_tls_states_t, previous_task) - 3080usize]; ["Offset of field: _jl_tls_states_t::root_task"] - [::std::mem::offset_of!(_jl_tls_states_t, root_task) - 2120usize]; + [::std::mem::offset_of!(_jl_tls_states_t, root_task) - 3088usize]; ["Offset of field: _jl_tls_states_t::timing_stack"] - [::std::mem::offset_of!(_jl_tls_states_t, timing_stack) - 2128usize]; + [::std::mem::offset_of!(_jl_tls_states_t, timing_stack) - 3096usize]; ["Offset of field: _jl_tls_states_t::stackbase"] - [::std::mem::offset_of!(_jl_tls_states_t, stackbase) - 2136usize]; + [::std::mem::offset_of!(_jl_tls_states_t, stackbase) - 3104usize]; ["Offset of field: _jl_tls_states_t::stacksize"] - [::std::mem::offset_of!(_jl_tls_states_t, stacksize) - 2144usize]; + [::std::mem::offset_of!(_jl_tls_states_t, stacksize) - 3112usize]; ["Offset of field: _jl_tls_states_t::sig_exception"] - [::std::mem::offset_of!(_jl_tls_states_t, sig_exception) - 2152usize]; + [::std::mem::offset_of!(_jl_tls_states_t, sig_exception) - 3120usize]; ["Offset of field: _jl_tls_states_t::bt_data"] - [::std::mem::offset_of!(_jl_tls_states_t, bt_data) - 2160usize]; + [::std::mem::offset_of!(_jl_tls_states_t, bt_data) - 3128usize]; ["Offset of field: _jl_tls_states_t::bt_size"] - [::std::mem::offset_of!(_jl_tls_states_t, bt_size) - 2168usize]; + [::std::mem::offset_of!(_jl_tls_states_t, bt_size) - 3136usize]; ["Offset of field: _jl_tls_states_t::profiling_bt_buffer"] - [::std::mem::offset_of!(_jl_tls_states_t, profiling_bt_buffer) - 2176usize]; + [::std::mem::offset_of!(_jl_tls_states_t, profiling_bt_buffer) - 3144usize]; ["Offset of field: _jl_tls_states_t::signal_request"] - [::std::mem::offset_of!(_jl_tls_states_t, signal_request) - 2184usize]; + [::std::mem::offset_of!(_jl_tls_states_t, signal_request) - 3152usize]; ["Offset of field: _jl_tls_states_t::io_wait"] - [::std::mem::offset_of!(_jl_tls_states_t, io_wait) - 2188usize]; + [::std::mem::offset_of!(_jl_tls_states_t, io_wait) - 3156usize]; ["Offset of field: _jl_tls_states_t::signal_stack"] - [::std::mem::offset_of!(_jl_tls_states_t, signal_stack) - 2192usize]; + [::std::mem::offset_of!(_jl_tls_states_t, signal_stack) - 3160usize]; ["Offset of field: _jl_tls_states_t::signal_stack_size"] - [::std::mem::offset_of!(_jl_tls_states_t, signal_stack_size) - 2200usize]; + [::std::mem::offset_of!(_jl_tls_states_t, signal_stack_size) - 3168usize]; ["Offset of field: _jl_tls_states_t::system_id"] - [::std::mem::offset_of!(_jl_tls_states_t, system_id) - 2208usize]; + [::std::mem::offset_of!(_jl_tls_states_t, system_id) - 3176usize]; ["Offset of field: _jl_tls_states_t::suspend_count"] - [::std::mem::offset_of!(_jl_tls_states_t, suspend_count) - 2216usize]; + [::std::mem::offset_of!(_jl_tls_states_t, suspend_count) - 3184usize]; ["Offset of field: _jl_tls_states_t::finalizers"] - [::std::mem::offset_of!(_jl_tls_states_t, finalizers) - 2224usize]; + [::std::mem::offset_of!(_jl_tls_states_t, finalizers) - 3192usize]; ["Offset of field: _jl_tls_states_t::previous_exception"] - [::std::mem::offset_of!(_jl_tls_states_t, previous_exception) - 2480usize]; + [::std::mem::offset_of!(_jl_tls_states_t, previous_exception) - 3448usize]; ["Offset of field: _jl_tls_states_t::locks"] - [::std::mem::offset_of!(_jl_tls_states_t, locks) - 2488usize]; + [::std::mem::offset_of!(_jl_tls_states_t, locks) - 3456usize]; ["Offset of field: _jl_tls_states_t::engine_nqueued"] - [::std::mem::offset_of!(_jl_tls_states_t, engine_nqueued) - 2552usize]; + [::std::mem::offset_of!(_jl_tls_states_t, engine_nqueued) - 3520usize]; }; pub type jl_function_t = jl_value_t; pub type jl_timing_block_t = _jl_timing_block_t; @@ -643,6 +777,7 @@ pub struct _jl_task_t { pub threadpoolid: i8, pub reentrant_timing: u8, pub gcstack: *mut jl_gcframe_t, + pub gcpreserve_stack: *mut jl_gcframe_t, pub world_age: usize, pub ptls: jl_ptls_t, pub excstack: *mut jl_excstack_t, @@ -651,7 +786,7 @@ pub struct _jl_task_t { } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of _jl_task_t"][::std::mem::size_of::<_jl_task_t>() - 224usize]; + ["Size of _jl_task_t"][::std::mem::size_of::<_jl_task_t>() - 232usize]; ["Alignment of _jl_task_t"][::std::mem::align_of::<_jl_task_t>() - 8usize]; ["Offset of field: _jl_task_t::next"][::std::mem::offset_of!(_jl_task_t, next) - 0usize]; ["Offset of field: _jl_task_t::queue"][::std::mem::offset_of!(_jl_task_t, queue) - 8usize]; @@ -688,13 +823,15 @@ const _: () = { [::std::mem::offset_of!(_jl_task_t, reentrant_timing) - 147usize]; ["Offset of field: _jl_task_t::gcstack"] [::std::mem::offset_of!(_jl_task_t, gcstack) - 152usize]; + ["Offset of field: _jl_task_t::gcpreserve_stack"] + [::std::mem::offset_of!(_jl_task_t, gcpreserve_stack) - 160usize]; ["Offset of field: _jl_task_t::world_age"] - [::std::mem::offset_of!(_jl_task_t, world_age) - 160usize]; - ["Offset of field: _jl_task_t::ptls"][::std::mem::offset_of!(_jl_task_t, ptls) - 168usize]; + [::std::mem::offset_of!(_jl_task_t, world_age) - 168usize]; + ["Offset of field: _jl_task_t::ptls"][::std::mem::offset_of!(_jl_task_t, ptls) - 176usize]; ["Offset of field: _jl_task_t::excstack"] - [::std::mem::offset_of!(_jl_task_t, excstack) - 176usize]; - ["Offset of field: _jl_task_t::eh"][::std::mem::offset_of!(_jl_task_t, eh) - 184usize]; - ["Offset of field: _jl_task_t::ctx"][::std::mem::offset_of!(_jl_task_t, ctx) - 192usize]; + [::std::mem::offset_of!(_jl_task_t, excstack) - 184usize]; + ["Offset of field: _jl_task_t::eh"][::std::mem::offset_of!(_jl_task_t, eh) - 192usize]; + ["Offset of field: _jl_task_t::ctx"][::std::mem::offset_of!(_jl_task_t, ctx) - 200usize]; }; pub type jl_task_t = _jl_task_t; #[repr(C)] @@ -1985,6 +2122,7 @@ pub struct _jl_module_t { pub bindingkeyset: u64, pub file: *mut jl_sym_t, pub line: i32, + pub usings_backedges: *mut jl_value_t, pub usings: arraylist_t, pub build_id: jl_uuid_t, pub uuid: jl_uuid_t, @@ -2000,7 +2138,7 @@ pub struct _jl_module_t { } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of _jl_module_t"][::std::mem::size_of::<_jl_module_t>() - 376usize]; + ["Size of _jl_module_t"][::std::mem::size_of::<_jl_module_t>() - 384usize]; ["Alignment of _jl_module_t"][::std::mem::align_of::<_jl_module_t>() - 8usize]; ["Offset of field: _jl_module_t::name"][::std::mem::offset_of!(_jl_module_t, name) - 0usize]; ["Offset of field: _jl_module_t::parent"] @@ -2011,27 +2149,29 @@ const _: () = { [::std::mem::offset_of!(_jl_module_t, bindingkeyset) - 24usize]; ["Offset of field: _jl_module_t::file"][::std::mem::offset_of!(_jl_module_t, file) - 32usize]; ["Offset of field: _jl_module_t::line"][::std::mem::offset_of!(_jl_module_t, line) - 40usize]; + ["Offset of field: _jl_module_t::usings_backedges"] + [::std::mem::offset_of!(_jl_module_t, usings_backedges) - 48usize]; ["Offset of field: _jl_module_t::usings"] - [::std::mem::offset_of!(_jl_module_t, usings) - 48usize]; + [::std::mem::offset_of!(_jl_module_t, usings) - 56usize]; ["Offset of field: _jl_module_t::build_id"] - [::std::mem::offset_of!(_jl_module_t, build_id) - 304usize]; - ["Offset of field: _jl_module_t::uuid"][::std::mem::offset_of!(_jl_module_t, uuid) - 320usize]; + [::std::mem::offset_of!(_jl_module_t, build_id) - 312usize]; + ["Offset of field: _jl_module_t::uuid"][::std::mem::offset_of!(_jl_module_t, uuid) - 328usize]; ["Offset of field: _jl_module_t::counter"] - [::std::mem::offset_of!(_jl_module_t, counter) - 336usize]; + [::std::mem::offset_of!(_jl_module_t, counter) - 344usize]; ["Offset of field: _jl_module_t::nospecialize"] - [::std::mem::offset_of!(_jl_module_t, nospecialize) - 340usize]; + [::std::mem::offset_of!(_jl_module_t, nospecialize) - 348usize]; ["Offset of field: _jl_module_t::optlevel"] - [::std::mem::offset_of!(_jl_module_t, optlevel) - 344usize]; + [::std::mem::offset_of!(_jl_module_t, optlevel) - 352usize]; ["Offset of field: _jl_module_t::compile"] - [::std::mem::offset_of!(_jl_module_t, compile) - 345usize]; + [::std::mem::offset_of!(_jl_module_t, compile) - 353usize]; ["Offset of field: _jl_module_t::infer"] - [::std::mem::offset_of!(_jl_module_t, infer) - 346usize]; + [::std::mem::offset_of!(_jl_module_t, infer) - 354usize]; ["Offset of field: _jl_module_t::istopmod"] - [::std::mem::offset_of!(_jl_module_t, istopmod) - 347usize]; + [::std::mem::offset_of!(_jl_module_t, istopmod) - 355usize]; ["Offset of field: _jl_module_t::max_methods"] - [::std::mem::offset_of!(_jl_module_t, max_methods) - 348usize]; - ["Offset of field: _jl_module_t::lock"][::std::mem::offset_of!(_jl_module_t, lock) - 352usize]; - ["Offset of field: _jl_module_t::hash"][::std::mem::offset_of!(_jl_module_t, hash) - 368usize]; + [::std::mem::offset_of!(_jl_module_t, max_methods) - 356usize]; + ["Offset of field: _jl_module_t::lock"][::std::mem::offset_of!(_jl_module_t, lock) - 360usize]; + ["Offset of field: _jl_module_t::hash"][::std::mem::offset_of!(_jl_module_t, hash) - 376usize]; }; pub type jl_module_t = _jl_module_t; #[repr(C)] @@ -2191,6 +2331,99 @@ const _: () = { ["Offset of field: _jl_excstack_t::reserved_size"] [::std::mem::offset_of!(_jl_excstack_t, reserved_size) - 8usize]; }; +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _bigval_t { + pub next: *mut _bigval_t, + pub prev: *mut _bigval_t, + pub sz: usize, + pub _padding: [*mut ::std::os::raw::c_void; 4usize], + pub __bindgen_anon_1: _bigval_t__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union _bigval_t__bindgen_ty_1 { + pub header: usize, + pub bits: _bigval_t__bindgen_ty_1__bindgen_ty_1, +} +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Copy, Clone)] +pub struct _bigval_t__bindgen_ty_1__bindgen_ty_1 { + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>, + pub __bindgen_padding_0: [u8; 7usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _bigval_t__bindgen_ty_1__bindgen_ty_1"] + [::std::mem::size_of::<_bigval_t__bindgen_ty_1__bindgen_ty_1>() - 8usize]; + ["Alignment of _bigval_t__bindgen_ty_1__bindgen_ty_1"] + [::std::mem::align_of::<_bigval_t__bindgen_ty_1__bindgen_ty_1>() - 8usize]; +}; +impl _bigval_t__bindgen_ty_1__bindgen_ty_1 { + #[inline] + pub fn gc(&self) -> usize { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 2u8) as u64) } + } + #[inline] + pub fn set_gc(&mut self, val: usize) { + unsafe { + let val: u64 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 2u8, val as u64) + } + } + #[inline] + pub unsafe fn gc_raw(this: *const Self) -> usize { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 2u8, + ) as u64) + } + } + #[inline] + pub unsafe fn set_gc_raw(this: *mut Self, val: usize) { + unsafe { + let val: u64 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 2u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1(gc: usize) -> __BindgenBitfieldUnit<[u8; 1usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 1usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 2u8, { + let gc: u64 = unsafe { ::std::mem::transmute(gc) }; + gc as u64 + }); + __bindgen_bitfield_unit + } +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _bigval_t__bindgen_ty_1"][::std::mem::size_of::<_bigval_t__bindgen_ty_1>() - 8usize]; + ["Alignment of _bigval_t__bindgen_ty_1"] + [::std::mem::align_of::<_bigval_t__bindgen_ty_1>() - 8usize]; + ["Offset of field: _bigval_t__bindgen_ty_1::header"] + [::std::mem::offset_of!(_bigval_t__bindgen_ty_1, header) - 0usize]; + ["Offset of field: _bigval_t__bindgen_ty_1::bits"] + [::std::mem::offset_of!(_bigval_t__bindgen_ty_1, bits) - 0usize]; +}; +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _bigval_t"][::std::mem::size_of::<_bigval_t>() - 64usize]; + ["Alignment of _bigval_t"][::std::mem::align_of::<_bigval_t>() - 8usize]; + ["Offset of field: _bigval_t::next"][::std::mem::offset_of!(_bigval_t, next) - 0usize]; + ["Offset of field: _bigval_t::prev"][::std::mem::offset_of!(_bigval_t, prev) - 8usize]; + ["Offset of field: _bigval_t::sz"][::std::mem::offset_of!(_bigval_t, sz) - 16usize]; + ["Offset of field: _bigval_t::_padding"][::std::mem::offset_of!(_bigval_t, _padding) - 24usize]; +}; #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { ["Size of template specialization: std_atomic_open0_size_t_close0"] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2d1ad00b..8414b780 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -22,6 +22,7 @@ pub mod api; mod build_info; pub mod collection; pub mod gc_trigger; +pub mod conservative; pub mod object_model; pub mod reference_glue; pub mod scanning; @@ -120,6 +121,33 @@ extern "C" { pub fn jl_gc_get_owner_address_to_mmtk(m: Address) -> Address; pub fn jl_gc_genericmemory_how(m: Address) -> usize; pub fn jl_gc_get_max_memory() -> usize; + pub fn jl_active_task_stack( + task: *const crate::julia_types::jl_task_t, + active_start: *mut Address, + active_end: *mut Address, + total_start: *mut Address, + total_end: *mut Address, + ); + pub static jl_true: *mut crate::julia_types::jl_value_t; +} + +#[macro_export] +macro_rules! early_return_for_non_moving_build { + ($ret_val:expr) => { + if cfg!(feature = "non_moving") { + return $ret_val; + } + }; +} + +/// Skip some methods if the current GC does not move objects +#[macro_export] +macro_rules! early_return_for_current_gc { + () => { + if !crate::collection::is_current_gc_moving() { + return; + } + }; } pub(crate) fn set_panic_hook() { diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 86a7b332..feefc983 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -21,15 +21,11 @@ pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBit pub(crate) const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::side_after(LOS_METADATA_SPEC.as_spec()); -#[cfg(feature = "object_pinning")] pub(crate) const LOCAL_PINNING_METADATA_BITS_SPEC: VMLocalPinningBitSpec = VMLocalPinningBitSpec::side_after(MARKING_METADATA_SPEC.as_spec()); -// pub(crate) const LOCAL_FORWARDING_POINTER_METADATA_SPEC: VMLocalForwardingPointerSpec = -// VMLocalForwardingPointerSpec::side_after(MARKING_METADATA_SPEC.as_spec()); - -// pub(crate) const LOCAL_FORWARDING_METADATA_BITS_SPEC: VMLocalForwardingBitsSpec = -// VMLocalForwardingBitsSpec::side_after(LOCAL_FORWARDING_POINTER_METADATA_SPEC.as_spec()); +pub(crate) const LOCAL_FORWARDING_METADATA_BITS_SPEC: VMLocalForwardingBitsSpec = + VMLocalForwardingBitsSpec::side_after(LOCAL_PINNING_METADATA_BITS_SPEC.as_spec()); /// PolicySpecific mark-and-nursery bits metadata spec /// 2-bits per object @@ -41,21 +37,15 @@ impl ObjectModel for VMObjectModel { const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = VMLocalForwardingPointerSpec::in_header(-64); - #[cfg(feature = "object_pinning")] - const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = - VMLocalForwardingBitsSpec::side_after(LOCAL_PINNING_METADATA_BITS_SPEC.as_spec()); - #[cfg(not(feature = "object_pinning"))] + const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = LOCAL_PINNING_METADATA_BITS_SPEC; const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = - VMLocalForwardingBitsSpec::side_after(MARKING_METADATA_SPEC.as_spec()); + LOCAL_FORWARDING_METADATA_BITS_SPEC; const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = MARKING_METADATA_SPEC; const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = LOS_METADATA_SPEC; const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = false; const OBJECT_REF_OFFSET_LOWER_BOUND: isize = 0; - #[cfg(feature = "object_pinning")] - const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = LOCAL_PINNING_METADATA_BITS_SPEC; - fn copy( from: ObjectReference, semantics: CopySemantics, @@ -123,10 +113,20 @@ impl ObjectModel for VMObjectModel { } fn get_current_size(object: ObjectReference) -> usize { - // not being called by objects in LOS - debug_assert!(!is_object_in_los(&object)); - - unsafe { get_so_object_size(object) } + if is_object_in_los(&object) { + unsafe { get_lo_object_size(object) } + } else if is_object_in_immixspace(&object) { + unsafe { get_so_object_size(object) } + } else { + // This is hacky but it should work. + // This covers the cases for immortal space and VM space. + // For those spaces, we only query object size when we try to find the base reference for an internal pointer. + // For those two spaces, we bulk set VO bits so we cannot find the base reference at all. + // We return 0 as the object size, so MMTk core won't find the base reference. + // As we only use the base reference to pin the objects, we cannot pin the objects. But it is fine, + // as objects in those spaces won't be moved. + 0 + } } fn get_size_when_copied(_object: ObjectReference) -> usize { @@ -176,7 +176,19 @@ pub fn is_object_in_los(object: &ObjectReference) -> bool { } #[inline(always)] +pub fn is_object_in_immixspace(object: &ObjectReference) -> bool { + is_addr_in_immixspace((*object).to_raw_address()) +} + +#[inline(always)] +pub fn is_addr_in_immixspace(addr: Address) -> bool { + // FIXME: get the range from MMTk. Or at least assert at boot time to make sure those constants are correct. + addr.as_usize() >= 0x200_0000_0000 && addr.as_usize() < 0x400_0000_0000 +} + /// This function uses mutable static variables and requires unsafe annotation + +#[inline(always)] pub unsafe fn get_so_object_size(object: ObjectReference) -> usize { let obj_address = object.to_raw_address(); let mut vtag = mmtk_jl_typetagof(obj_address); @@ -303,6 +315,13 @@ pub unsafe fn get_so_object_size(object: ObjectReference) -> usize { llt_align(dtsz + JULIA_HEADER_SIZE, 16) } +#[inline(always)] +pub unsafe fn get_lo_object_size(object: ObjectReference) -> usize { + let obj_address = object.to_raw_address(); + let julia_big_object = (obj_address - std::mem::size_of::<_bigval_t>()).to_ptr::<_bigval_t>(); + return (*julia_big_object).sz; +} + #[inline(always)] pub unsafe fn get_object_start_ref(object: ObjectReference) -> Address { let obj_address = object.to_raw_address(); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index a82209ed..34addb08 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -53,15 +53,44 @@ impl Scanning for VMScanning { use mmtk::util::Address; let ptls: &mut _jl_tls_states_t = unsafe { std::mem::transmute(mutator.mutator_tls) }; - let mut slot_buffer = SlotBuffer { buffer: vec![] }; // need to be tpinned as they're all from the shadow stack + let pthread = ptls.system_id; + let mut tpinning_slot_buffer = SlotBuffer { buffer: vec![] }; // need to be transitively pinned + let mut pinning_slot_buffer = SlotBuffer { buffer: vec![] }; // roots from the shadow stack that we know that do not need to be transitively pinned let mut node_buffer = vec![]; + // Conservatively scan registers saved with the thread + crate::conservative::mmtk_conservative_scan_ptls_registers(ptls); + // Scan thread local from ptls: See gc_queue_thread_local in gc.c let mut root_scan_task = |task: *const _jl_task_t, task_is_root: bool| { if !task.is_null() { + // Scan gc preserve and shadow stacks unsafe { - crate::julia_scanning::mmtk_scan_gcstack(task, &mut slot_buffer); + crate::julia_scanning::mmtk_scan_gcpreserve_stack( + task, + &mut tpinning_slot_buffer, + ); + crate::julia_scanning::mmtk_scan_gcstack( + task, + &mut tpinning_slot_buffer, + Some(&mut pinning_slot_buffer), + ); } + + // Conservatively scan native stacks to make sure we won't move objects that the runtime is using. + log::debug!( + "Scanning ptls {:?}, pthread {:x}", + mutator.mutator_tls, + pthread + ); + // Conservative scan stack and registers. If the task hasn't been started, we do not need to scan its stack and registers. + // We cannot use `task->start` to skip conservative scanning, as before a task is started, the runtime may evaluate the code and we need to make sure the runtime objects are properly scanned. + // However, without this check, we may timeout in a test that spawns a lot of tasks. + // if unsafe { (*task).start == crate::jl_true } { + crate::conservative::mmtk_conservative_scan_task_stack(task); + crate::conservative::mmtk_conservative_scan_task_registers(task); + // } + if task_is_root { // captures wrong root nodes before creating the work debug_assert!( @@ -136,13 +165,20 @@ impl Scanning for VMScanning { // Push work const CAPACITY_PER_PACKET: usize = 4096; - for tpinning_roots in slot_buffer + for tpinning_roots in tpinning_slot_buffer .buffer .chunks(CAPACITY_PER_PACKET) .map(|c| c.to_vec()) { factory.create_process_tpinning_roots_work(tpinning_roots); } + for pinning_roots in pinning_slot_buffer + .buffer + .chunks(CAPACITY_PER_PACKET) + .map(|c| c.to_vec()) + { + factory.create_process_pinning_roots_work(pinning_roots); + } for nodes in node_buffer.chunks(CAPACITY_PER_PACKET).map(|c| c.to_vec()) { factory.create_process_pinning_roots_work(nodes); } @@ -167,6 +203,9 @@ impl Scanning for VMScanning { process_object(object, slot_visitor); } fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { + // pin conservative roots from stack scanning + crate::conservative::pin_conservative_roots(); + let sweep_vm_specific_work = SweepVMSpecific::new(); memory_manager::add_work_packet( &SINGLETON,