diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 8702c17..13e28b8 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -187,9 +187,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "enum-map" -version = "2.6.3" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c188012f8542dee7b3996e44dd89461d64aa471b0a7c71a1ae2f595d259e96e5" +checksum = "53337c2dbf26a3c31eccc73a37b10c1614e8d4ae99b6a50d553e8936423c1f16" dependencies = [ "enum-map-derive", ] @@ -351,9 +351,9 @@ checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -383,7 +383,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=57af17fbfd94ff0df2cd3b1e504abe299ce4f0ab#57af17fbfd94ff0df2cd3b1e504abe299ce4f0ab" +source = "git+https://github.com/wks/mmtk-core.git?branch=feature/vm-forwarding#d0ac4dd94eda717bd438337e9efa78b5c66e0d3f" dependencies = [ "atomic", "atomic-traits", @@ -417,7 +417,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=57af17fbfd94ff0df2cd3b1e504abe299ce4f0ab#57af17fbfd94ff0df2cd3b1e504abe299ce4f0ab" +source = "git+https://github.com/wks/mmtk-core.git?branch=feature/vm-forwarding#d0ac4dd94eda717bd438337e9efa78b5c66e0d3f" dependencies = [ "proc-macro-error", "proc-macro2", @@ -498,9 +498,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" +checksum = "b559898e0b4931ed2d3b959ab0c2da4d99cc644c4b0b1a35b4d344027f474023" [[package]] name = "probe" @@ -572,9 +572,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaac441002f822bc9705a681810a4dd2963094b9ca0ddc41cb963a4c189189ea" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", @@ -584,9 +584,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5011c7e263a695dc8ca064cddb722af1be54e517a280b12a5356f98366899e5d" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", @@ -610,9 +610,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.19" +version = "0.38.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" +checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" dependencies = [ "bitflags 2.4.1", "errno", @@ -762,18 +762,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 50f1675..44bfb3b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -33,12 +33,16 @@ atomic_refcell = "0.1.9" probe = "0.5" [dependencies.mmtk] -features = ["is_mmtk_object", "object_pinning"] +features = ["is_mmtk_object", "object_pinning", "vm_forwarding"] -# Uncomment one of the following lines to choose where to find mmtk-core. -git = "https://github.com/mmtk/mmtk-core.git" # Use mmtk-core from the official repository. -#path = "../../mmtk-core" # Use mmtk-core from a local repository. -rev = "57af17fbfd94ff0df2cd3b1e504abe299ce4f0ab" +# Uncomment the following two lines to use the given revision of mmtk-core from the official repository. +#git = "https://github.com/mmtk/mmtk-core.git" +#rev = "57af17fbfd94ff0df2cd3b1e504abe299ce4f0ab" +git = "https://github.com/wks/mmtk-core.git" +branch = "feature/vm-forwarding" + +# Uncomment the following line to use mmtk-core from a local repository. +#path = "../../mmtk-core" [features] default = [] diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index e7640b0..52c02c3 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,3 +1,5 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + use crate::api::RubyMutator; use crate::{upcalls, Ruby}; use mmtk::scheduler::{GCController, GCWorker}; @@ -13,6 +15,48 @@ pub const GC_THREAD_KIND_WORKER: libc::c_int = 1; const HAS_MOVED_GIVTBL: usize = 1 << 63; const HIDDEN_SIZE_MASK: usize = 0x0000FFFFFFFFFFFF; +const BUILTIN_TYPE_MASK: usize = 0x1f; +const T_MOVED: usize = 0x1e; + +const RUBY_FL_PROMOTED: usize = 1 << 5; +const RUBY_FL_EXIVAR: usize = 1 << 10; + +fn flags_is_forwarding_not_triggered_yet(flags: usize) -> bool { + (flags & BUILTIN_TYPE_MASK) != T_MOVED && (flags & RUBY_FL_PROMOTED) == 0 +} + +fn flags_is_being_forwarded(flags: usize) -> bool { + (flags & BUILTIN_TYPE_MASK) != T_MOVED && (flags & RUBY_FL_PROMOTED) != 0 +} + +fn flags_is_forwarded(flags: usize) -> bool { + (flags & BUILTIN_TYPE_MASK) == T_MOVED +} + +fn flags_is_forwarded_or_being_forwarded(flags: usize) -> bool { + !flags_is_forwarding_not_triggered_yet(flags) +} + +fn flags_set_being_forwarded(flags: usize) -> usize { + flags | RUBY_FL_PROMOTED +} + +pub fn flags_clear_being_forwarded(flags: usize) -> usize { + flags & !RUBY_FL_PROMOTED +} + +fn flags_set_forwarded_with_forwarding_pointer(forwarding_pointer: usize) -> usize { + T_MOVED | forwarding_pointer << 8 +} + +fn flags_get_forwarding_pointer(flags: usize) -> usize { + flags >> 8 +} + +pub fn flags_has_fl_exivar(flags: usize) -> bool { + flags & RUBY_FL_EXIVAR != 0 +} + /// Provide convenient methods for accessing Ruby objects. /// TODO: Wrap C functions in `RubyUpcalls` as Rust-friendly methods. pub struct RubyObjectAccess { @@ -124,6 +168,104 @@ impl RubyObjectAccess { Some(addr) } } + + pub fn flags_field_atomic(&self) -> &'static AtomicUsize { + unsafe { self.objref.to_raw_address().as_ref::() } + } + + pub fn attempt_to_forward(&self) -> bool { + trace!("attempt_to_forward({})", self.objref); + let flags = self.flags_field_atomic(); + + let old_value = flags.load(Ordering::Relaxed); + if flags_is_forwarding_not_triggered_yet(old_value) { + trace!("Forwarding not triggered, yet. old value: {:x}", old_value); + // Forwarding not triggered yet. Try to transition the state. + let new_value = flags_set_being_forwarded(old_value); + + match flags.compare_exchange(old_value, new_value, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(actual) => { + debug_assert_eq!(actual, old_value); + true + } + Err(actual) => { + debug_assert_ne!(actual, old_value); + debug_assert!(flags_is_forwarded_or_being_forwarded(actual)); + false + } + } + } else { + trace!("Already forwarded. old value: {:x}", old_value); + // Already forwarded. Fail. + false + } + } + + pub fn write_forwarding_state_and_forwarding_pointer(&self, new_object: ObjectReference) { + trace!( + "write_forwarding_state_and_forwarding_pointer({}, {})", + self.objref, + new_object + ); + + let flags = self.flags_field_atomic(); + // Note: no need for atomic RMW operation because we currently owns this object. + let new_value = + flags_set_forwarded_with_forwarding_pointer(new_object.to_raw_address().as_usize()); + // Ordering matters. The following store is a release store. + flags.store(new_value, Ordering::Release); + trace!("Obj: {} Forwarding complete!", self.objref); + } + + pub fn revert_forwarding_state(&self) { + trace!("revert_forwarding_state({})", self.objref); + let flags = self.flags_field_atomic(); + // Note: no need for atomic RMW operation because we currently owns this object. + let old_value = flags.load(Ordering::Relaxed); + debug_assert!(flags_is_being_forwarded(old_value)); + let new_value = flags_clear_being_forwarded(old_value); + flags.store(new_value, Ordering::Relaxed); + } + + pub fn spin_and_get_forwarded_object(&self) -> ObjectReference { + trace!("spin_and_get_forwarded_object({})", self.objref); + let flags = self.flags_field_atomic(); + + loop { + let value = flags.load(Ordering::Acquire); + // Ordering matters. The load above is an acquire load. + + trace!("Obj: {} Spinning... Old value: {:x}", self.objref, value); + + if flags_is_forwarded(value) { + trace!("Obj: {} Complete. Loading fwd ptr...", self.objref); + // Load forwarding pointer. + break self.load_forwarding_pointer(); + } else if flags_is_being_forwarded(value) { + // Being forwarded. + // Keep waiting + } else { + trace!("Obj: {} Reverted. Loading fwd ptr...", self.objref); + // Reverted. Return self. + break self.objref; + } + } + } + + pub fn is_forwarded(&self) -> bool { + let flags = self.flags_field_atomic(); + let old_value = flags.load(Ordering::Relaxed); + flags_is_forwarded(old_value) + } + + pub fn load_forwarding_pointer(&self) -> ObjectReference { + let flags = self.flags_field_atomic(); + let old_value = flags.load(Ordering::Relaxed); + ObjectReference::from_raw_address(unsafe { + Address::from_usize(flags_get_forwarding_pointer(old_value)) + }) + } } type ObjectClosureFunction = diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index db44111..5c58f32 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,8 +1,8 @@ use std::ptr::copy_nonoverlapping; +use std::sync::atomic::Ordering; -use crate::abi::{RubyObjectAccess, MIN_OBJ_ALIGN, OBJREF_OFFSET}; +use crate::abi::{flags_has_fl_exivar, RubyObjectAccess, MIN_OBJ_ALIGN, OBJREF_OFFSET, flags_clear_being_forwarded}; use crate::{abi, Ruby}; -use mmtk::util::constants::BITS_IN_BYTE; use mmtk::util::copy::{CopySemantics, GCWorkerCopyContext}; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::*; @@ -14,17 +14,21 @@ impl VMObjectModel { } impl ObjectModel for VMObjectModel { + type VMForwardingDataType = (); + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); - // We overwrite the prepended word which were used to hold object sizes. + /// Not used. We implement forwarding in the VM binding, and put the forwarding pointer at the + /// same offset as `RMoved::destination` in C. const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = - VMLocalForwardingPointerSpec::in_header(-((OBJREF_OFFSET * BITS_IN_BYTE) as isize)); + VMLocalForwardingPointerSpec::in_header(0); + /// Not used. We implement forwarding in the VM binding, and represent forwarding states with + /// `T_MOVED`. See `abi.rs`. const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = - VMLocalForwardingBitsSpec::side_first(); + VMLocalForwardingBitsSpec::in_header(0); - const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = - VMLocalMarkBitSpec::side_after(Self::LOCAL_FORWARDING_BITS_SPEC.as_spec()); + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::side_first(); const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = VMLocalPinningBitSpec::side_after(Self::LOCAL_MARK_BIT_SPEC.as_spec()); @@ -41,9 +45,16 @@ impl ObjectModel for VMObjectModel { from: ObjectReference, semantics: CopySemantics, copy_context: &mut GCWorkerCopyContext, + _vm_data: Self::VMForwardingDataType, ) -> ObjectReference { let from_acc = RubyObjectAccess::from_objref(from); - let maybe_givtbl = from_acc.get_original_givtbl(); + let flags = from_acc.flags_field_atomic(); + let old_flags = flags.load(Ordering::Relaxed); + let maybe_givtbl = if flags_has_fl_exivar(old_flags) { + from_acc.get_original_givtbl() + } else { + None + }; let from_start = from_acc.obj_start(); let object_size = from_acc.object_size(); let to_start = copy_context.alloc_copy(from, object_size, MIN_OBJ_ALIGN, 0, semantics); @@ -51,6 +62,14 @@ impl ObjectModel for VMObjectModel { unsafe { copy_nonoverlapping::(from_start.to_ptr(), to_start.to_mut_ptr(), object_size); } + + // The `flags` field of the from-space copy is overwritten to `T_MOVED`. + // Reconstruct the `flags` of the to-space copy. + let new_flags = flags_clear_being_forwarded(old_flags); + unsafe { + to_payload.store::(new_flags); + } + let to_obj = ObjectReference::from_raw_address(to_payload); copy_context.post_copy(to_obj, object_size, semantics); trace!("Copied object from {} to {}", from, to_obj); @@ -81,6 +100,7 @@ impl ObjectModel for VMObjectModel { } let to_acc = RubyObjectAccess::from_objref(to_obj); to_acc.set_has_moved_givtbl(); + warn!("{} has moved givtbl", to_obj); } to_obj @@ -137,4 +157,32 @@ impl ObjectModel for VMObjectModel { fn dump_object(_object: ObjectReference) { todo!() } + + fn attempt_to_forward(object: ObjectReference) -> Option { + RubyObjectAccess::from_objref(object).attempt_to_forward().then_some(()) + } + + fn write_forwarding_state_and_forwarding_pointer( + object: ObjectReference, + new_object: ObjectReference, + ) { + RubyObjectAccess::from_objref(object) + .write_forwarding_state_and_forwarding_pointer(new_object) + } + + fn revert_forwarding_state(object: ObjectReference, _vm_data: Self::VMForwardingDataType) { + RubyObjectAccess::from_objref(object).revert_forwarding_state() + } + + fn spin_and_get_forwarded_object(object: ObjectReference) -> ObjectReference { + RubyObjectAccess::from_objref(object).spin_and_get_forwarded_object() + } + + fn is_forwarded(object: ObjectReference) -> bool { + RubyObjectAccess::from_objref(object).is_forwarded() + } + + fn read_forwarding_pointer(object: ObjectReference) -> ObjectReference { + RubyObjectAccess::from_objref(object).load_forwarding_pointer() + } }