From ab40b2b342adcd89004186135b31f49422521fb9 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 19 Oct 2023 16:38:51 +0800 Subject: [PATCH 01/10] WIP: Implementing VM-side obj forwarding --- mmtk/Cargo.lock | 2 -- mmtk/Cargo.toml | 4 ++-- mmtk/src/object_model.rs | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index dbd7c24..e7d4ac3 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -383,7 +383,6 @@ dependencies = [ [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/mmtk/mmtk-core.git#29d482cf7b470ad3b09f88c6993babd6f695d832" dependencies = [ "atomic", "atomic-traits", @@ -417,7 +416,6 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/mmtk/mmtk-core.git#29d482cf7b470ad3b09f88c6993babd6f695d832" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 73a0554..767ec61 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -36,8 +36,8 @@ probe = "0.5" features = ["is_mmtk_object", "object_pinning"] # Uncomment one of the following lines to choose where to find mmtk-core. -git = "https://github.com/mmtk/mmtk-core.git" # Use mmtk-core from the official repository. -#path = "../../mmtk-core" # Use mmtk-core from a local repository. +#git = "https://github.com/mmtk/mmtk-core.git" # Use mmtk-core from the official repository. +path = "../../mmtk-core" # Use mmtk-core from a local repository. [features] default = [] diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index db44111..8f8e72c 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -37,6 +37,8 @@ impl ObjectModel for VMObjectModel { const NEED_VO_BITS_DURING_TRACING: bool = true; + const VM_IMPLEMENTED_FORWARDING: bool = true; + fn copy( from: ObjectReference, semantics: CopySemantics, @@ -137,4 +139,25 @@ impl ObjectModel for VMObjectModel { fn dump_object(_object: ObjectReference) { todo!() } + + fn attempt_to_forward(object: ObjectReference) -> Result { + + } + + fn write_forwarding_state_and_forwarding_pointer( + object: ObjectReference, + new_object: ObjectReference, + ) { + + } + + fn revert_forwarding_state(object: ObjectReference, vm_data: usize) { + + } + + fn spin_and_get_forwarded_object( + object: ObjectReference, + ) -> ObjectReference { + + } } From 4c587ae9bbe2ee1ab8e11cefb652631732cb8603 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 19 Oct 2023 19:35:01 +0800 Subject: [PATCH 02/10] WIP: Preliminary impl of VM-side forwarding --- mmtk/src/abi.rs | 96 ++++++++++++++++++++++++++++++++++++++++ mmtk/src/object_model.rs | 19 ++++---- 2 files changed, 105 insertions(+), 10 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index e7640b0..5464835 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,3 +1,5 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + use crate::api::RubyMutator; use crate::{upcalls, Ruby}; use mmtk::scheduler::{GCController, GCWorker}; @@ -13,6 +15,12 @@ pub const GC_THREAD_KIND_WORKER: libc::c_int = 1; const HAS_MOVED_GIVTBL: usize = 1 << 63; const HIDDEN_SIZE_MASK: usize = 0x0000FFFFFFFFFFFF; +const BUILTIN_TYPE_MASK: usize = 0x1f; +const T_MOVED: usize = 0x1e; +const T_MOVED_FORWARDED: usize = 0x3e; + +const FORWARDING_POINTER_OFFSET: usize = 16; + /// Provide convenient methods for accessing Ruby objects. /// TODO: Wrap C functions in `RubyUpcalls` as Rust-friendly methods. pub struct RubyObjectAccess { @@ -124,6 +132,94 @@ impl RubyObjectAccess { Some(addr) } } + + pub fn attempt_to_forward(&self) -> Result { + trace!("attempt_to_forward({})", self.objref); + let flags = unsafe { self.objref.to_raw_address().as_ref::() }; + + let old_value = flags.load(Ordering::Relaxed); + if old_value & BUILTIN_TYPE_MASK != T_MOVED { + trace!("Not forwarded, yet. old value: {:x}", old_value); + // Not forwarded yet + let new_value = T_MOVED; + + match flags.compare_exchange(old_value, new_value, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(actual) => { + debug_assert_eq!(actual, old_value); + Ok(old_value) + } + Err(actual) => { + debug_assert_ne!(actual, old_value); + debug_assert_eq!(actual & BUILTIN_TYPE_MASK, T_MOVED); + Err(()) + } + } + } else { + trace!("Already forwarded. old value: {:x}", old_value); + // Already forwarded. Fail. + Err(()) + } + } + + pub fn write_forwarding_state_and_forwarding_pointer( + &self, + new_object: ObjectReference, + ) { + trace!("write_forwarding_state_and_forwarding_pointer({}, {})", self.objref, new_object); + unsafe { + (self.objref.to_raw_address() + FORWARDING_POINTER_OFFSET).store::(new_object) + } + + // Ordering matters. The following store is a release store. + + let flags = unsafe { self.objref.to_raw_address().as_ref::() }; + flags.store(T_MOVED_FORWARDED, Ordering::Release); + trace!("Obj: {} Forwarding complete!", self.objref); + } + + pub fn revert_forwarding_state(&self, vm_data: usize) { + trace!("revert_forwarding_state({}, {:x})", self.objref, vm_data); + let flags = unsafe { self.objref.to_raw_address().as_ref::() }; + flags.store(vm_data, Ordering::Relaxed); + } + + pub fn spin_and_get_forwarded_object(&self) -> ObjectReference { + trace!("spin_and_get_forwarded_object({})", self.objref); + let flags = unsafe { self.objref.to_raw_address().as_ref::() }; + + loop { + let value = flags.load(Ordering::Acquire); + + trace!("Obj: {} Spinning... Old value: {:x}", self.objref, value); + + // Ordering matters. The load above is an acquire load. + + match value { + T_MOVED_FORWARDED => { + trace!("Obj: {} Complete. Loading fwd ptr...", self.objref); + // Forwarded. Load forwarding pointer. + break self.load_forwarding_pointer(); + } + T_MOVED => { + panic!("Obj: {} Being forwarded. How can this be! There is only one thread!", self.objref); + // Being forwarded. + // Keep waiting + } + _ => { + trace!("Obj: {} Reverted. Loading fwd ptr...", self.objref); + // Reverted. Return self. + break self.objref; + } + } + } + } + + pub fn load_forwarding_pointer(&self) -> ObjectReference { + unsafe { + (self.objref.to_raw_address() + FORWARDING_POINTER_OFFSET).load::() + } + } } type ObjectClosureFunction = diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 8f8e72c..0065747 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -141,23 +141,22 @@ impl ObjectModel for VMObjectModel { } fn attempt_to_forward(object: ObjectReference) -> Result { - + RubyObjectAccess::from_objref(object).attempt_to_forward() } fn write_forwarding_state_and_forwarding_pointer( - object: ObjectReference, - new_object: ObjectReference, - ) { - + object: ObjectReference, + new_object: ObjectReference, + ) { + RubyObjectAccess::from_objref(object) + .write_forwarding_state_and_forwarding_pointer(new_object) } fn revert_forwarding_state(object: ObjectReference, vm_data: usize) { - + RubyObjectAccess::from_objref(object).revert_forwarding_state(vm_data) } - fn spin_and_get_forwarded_object( - object: ObjectReference, - ) -> ObjectReference { - + fn spin_and_get_forwarded_object(object: ObjectReference) -> ObjectReference { + RubyObjectAccess::from_objref(object).spin_and_get_forwarded_object() } } From 812cbb430c4143dc7fbb45da5cbcf0eed60883dc Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 20 Oct 2023 15:23:07 +0800 Subject: [PATCH 03/10] Upstream API change --- mmtk/Cargo.toml | 2 +- mmtk/src/abi.rs | 18 ++++++++++++------ mmtk/src/object_model.rs | 22 +++++++++++++++++++--- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 767ec61..68254e4 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -33,7 +33,7 @@ atomic_refcell = "0.1.9" probe = "0.5" [dependencies.mmtk] -features = ["is_mmtk_object", "object_pinning"] +features = ["is_mmtk_object", "object_pinning", "vm_forwarding"] # Uncomment one of the following lines to choose where to find mmtk-core. #git = "https://github.com/mmtk/mmtk-core.git" # Use mmtk-core from the official repository. diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 5464835..57bfa1a 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -133,32 +133,32 @@ impl RubyObjectAccess { } } - pub fn attempt_to_forward(&self) -> Result { + pub fn attempt_to_forward(&self) -> Option { trace!("attempt_to_forward({})", self.objref); let flags = unsafe { self.objref.to_raw_address().as_ref::() }; let old_value = flags.load(Ordering::Relaxed); if old_value & BUILTIN_TYPE_MASK != T_MOVED { - trace!("Not forwarded, yet. old value: {:x}", old_value); - // Not forwarded yet + trace!("Forwarding not triggered, yet. old value: {:x}", old_value); + // Forwarding not triggered yet. Try to transition the state. let new_value = T_MOVED; match flags.compare_exchange(old_value, new_value, Ordering::Relaxed, Ordering::Relaxed) { Ok(actual) => { debug_assert_eq!(actual, old_value); - Ok(old_value) + Some(old_value) } Err(actual) => { debug_assert_ne!(actual, old_value); debug_assert_eq!(actual & BUILTIN_TYPE_MASK, T_MOVED); - Err(()) + None } } } else { trace!("Already forwarded. old value: {:x}", old_value); // Already forwarded. Fail. - Err(()) + None } } @@ -215,6 +215,12 @@ impl RubyObjectAccess { } } + pub fn is_forwarded(&self) -> bool { + let flags = unsafe { self.objref.to_raw_address().as_ref::() }; + let old_value = flags.load(Ordering::Relaxed); + old_value == T_MOVED_FORWARDED + } + pub fn load_forwarding_pointer(&self) -> ObjectReference { unsafe { (self.objref.to_raw_address() + FORWARDING_POINTER_OFFSET).load::() diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 0065747..794fa45 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -14,6 +14,8 @@ impl VMObjectModel { } impl ObjectModel for VMObjectModel { + type VMForwardingDataType = usize; + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); // We overwrite the prepended word which were used to hold object sizes. @@ -37,12 +39,11 @@ impl ObjectModel for VMObjectModel { const NEED_VO_BITS_DURING_TRACING: bool = true; - const VM_IMPLEMENTED_FORWARDING: bool = true; - fn copy( from: ObjectReference, semantics: CopySemantics, copy_context: &mut GCWorkerCopyContext, + vm_data: Self::VMForwardingDataType, ) -> ObjectReference { let from_acc = RubyObjectAccess::from_objref(from); let maybe_givtbl = from_acc.get_original_givtbl(); @@ -53,6 +54,13 @@ impl ObjectModel for VMObjectModel { unsafe { copy_nonoverlapping::(from_start.to_ptr(), to_start.to_mut_ptr(), object_size); } + + // The `flags` field of the from-space copy is overwritten to `T_MOVED`. + // Reconstruct the `flags` of the to-space copy. + unsafe { + to_payload.store::(vm_data); + } + let to_obj = ObjectReference::from_raw_address(to_payload); copy_context.post_copy(to_obj, object_size, semantics); trace!("Copied object from {} to {}", from, to_obj); @@ -140,7 +148,7 @@ impl ObjectModel for VMObjectModel { todo!() } - fn attempt_to_forward(object: ObjectReference) -> Result { + fn attempt_to_forward(object: ObjectReference) -> Option { RubyObjectAccess::from_objref(object).attempt_to_forward() } @@ -159,4 +167,12 @@ impl ObjectModel for VMObjectModel { fn spin_and_get_forwarded_object(object: ObjectReference) -> ObjectReference { RubyObjectAccess::from_objref(object).spin_and_get_forwarded_object() } + + fn is_forwarded(object: ObjectReference) -> bool { + RubyObjectAccess::from_objref(object).is_forwarded() + } + + fn read_forwarding_pointer(object: ObjectReference) -> ObjectReference { + RubyObjectAccess::from_objref(object).load_forwarding_pointer() + } } From 707ae16abfa16dd36a09a5db52b21e729f4bd3cc Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 20 Oct 2023 15:41:05 +0800 Subject: [PATCH 04/10] Deactivate fwd bits and fwd ptr metadata --- mmtk/src/object_model.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 794fa45..7b0e7b6 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -2,7 +2,6 @@ use std::ptr::copy_nonoverlapping; use crate::abi::{RubyObjectAccess, MIN_OBJ_ALIGN, OBJREF_OFFSET}; use crate::{abi, Ruby}; -use mmtk::util::constants::BITS_IN_BYTE; use mmtk::util::copy::{CopySemantics, GCWorkerCopyContext}; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::*; @@ -18,15 +17,18 @@ impl ObjectModel for VMObjectModel { const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); - // We overwrite the prepended word which were used to hold object sizes. + /// Not used. We implement forwarding in the VM binding, and put the forwarding pointer at the + /// same offset as `RMoved::destination` in C. const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = - VMLocalForwardingPointerSpec::in_header(-((OBJREF_OFFSET * BITS_IN_BYTE) as isize)); + VMLocalForwardingPointerSpec::in_header(0); + /// Not used. We implement forwarding in the VM binding, and represent forwarding states with + /// `T_MOVED`. See `abi.rs`. const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = - VMLocalForwardingBitsSpec::side_first(); + VMLocalForwardingBitsSpec::in_header(0); const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = - VMLocalMarkBitSpec::side_after(Self::LOCAL_FORWARDING_BITS_SPEC.as_spec()); + VMLocalMarkBitSpec::side_first(); const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = VMLocalPinningBitSpec::side_after(Self::LOCAL_MARK_BIT_SPEC.as_spec()); From 9b424c3ae7cf5519274d794e24dbd3f58dd173ee Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 24 Oct 2023 14:20:10 +0800 Subject: [PATCH 05/10] Remove the `panic!()` for debugging --- mmtk/src/abi.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 57bfa1a..2b5a06a 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -202,7 +202,6 @@ impl RubyObjectAccess { break self.load_forwarding_pointer(); } T_MOVED => { - panic!("Obj: {} Being forwarded. How can this be! There is only one thread!", self.objref); // Being forwarded. // Keep waiting } From 5906e6f2d02201a7c1eed0fb8f7d6ac794ef7159 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 24 Oct 2023 14:20:41 +0800 Subject: [PATCH 06/10] Formatting --- mmtk/src/abi.rs | 14 ++++++++------ mmtk/src/object_model.rs | 3 +-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 2b5a06a..4709d07 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -162,13 +162,15 @@ impl RubyObjectAccess { } } - pub fn write_forwarding_state_and_forwarding_pointer( - &self, - new_object: ObjectReference, - ) { - trace!("write_forwarding_state_and_forwarding_pointer({}, {})", self.objref, new_object); + pub fn write_forwarding_state_and_forwarding_pointer(&self, new_object: ObjectReference) { + trace!( + "write_forwarding_state_and_forwarding_pointer({}, {})", + self.objref, + new_object + ); unsafe { - (self.objref.to_raw_address() + FORWARDING_POINTER_OFFSET).store::(new_object) + (self.objref.to_raw_address() + FORWARDING_POINTER_OFFSET) + .store::(new_object) } // Ordering matters. The following store is a release store. diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 7b0e7b6..f1c1525 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -27,8 +27,7 @@ impl ObjectModel for VMObjectModel { const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = VMLocalForwardingBitsSpec::in_header(0); - const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = - VMLocalMarkBitSpec::side_first(); + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::side_first(); const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = VMLocalPinningBitSpec::side_after(Self::LOCAL_MARK_BIT_SPEC.as_spec()); From d849698ded26022ffab1b275d3e8eaece938d5ac Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 24 Oct 2023 14:28:20 +0800 Subject: [PATCH 07/10] Use GitHub repo of the mmtk-core PR --- mmtk/Cargo.lock | 2 ++ mmtk/Cargo.toml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 597e96a..13e28b8 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -383,6 +383,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.20.0" +source = "git+https://github.com/wks/mmtk-core.git?branch=feature/vm-forwarding#d0ac4dd94eda717bd438337e9efa78b5c66e0d3f" dependencies = [ "atomic", "atomic-traits", @@ -416,6 +417,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" +source = "git+https://github.com/wks/mmtk-core.git?branch=feature/vm-forwarding#d0ac4dd94eda717bd438337e9efa78b5c66e0d3f" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 85e704f..44bfb3b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -38,9 +38,11 @@ features = ["is_mmtk_object", "object_pinning", "vm_forwarding"] # Uncomment the following two lines to use the given revision of mmtk-core from the official repository. #git = "https://github.com/mmtk/mmtk-core.git" #rev = "57af17fbfd94ff0df2cd3b1e504abe299ce4f0ab" +git = "https://github.com/wks/mmtk-core.git" +branch = "feature/vm-forwarding" # Uncomment the following line to use mmtk-core from a local repository. -path = "../../mmtk-core" +#path = "../../mmtk-core" [features] default = [] From fdfc59cb11bcba0f34815a78a590498a30a2e4d1 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 27 Oct 2023 10:52:21 +0800 Subject: [PATCH 08/10] Test FL_EXIVAR in `copy`. --- mmtk/src/abi.rs | 6 ++++++ mmtk/src/object_model.rs | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 4709d07..003df09 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -21,6 +21,12 @@ const T_MOVED_FORWARDED: usize = 0x3e; const FORWARDING_POINTER_OFFSET: usize = 16; +const RUBY_FL_EXIVAR: usize = 1 << 10; + +pub fn flags_has_fl_exivar(flags: usize) -> bool { + flags & RUBY_FL_EXIVAR != 0 +} + /// Provide convenient methods for accessing Ruby objects. /// TODO: Wrap C functions in `RubyUpcalls` as Rust-friendly methods. pub struct RubyObjectAccess { diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index f1c1525..0fcaddb 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,6 +1,6 @@ use std::ptr::copy_nonoverlapping; -use crate::abi::{RubyObjectAccess, MIN_OBJ_ALIGN, OBJREF_OFFSET}; +use crate::abi::{flags_has_fl_exivar, RubyObjectAccess, MIN_OBJ_ALIGN, OBJREF_OFFSET}; use crate::{abi, Ruby}; use mmtk::util::copy::{CopySemantics, GCWorkerCopyContext}; use mmtk::util::{Address, ObjectReference}; @@ -47,7 +47,11 @@ impl ObjectModel for VMObjectModel { vm_data: Self::VMForwardingDataType, ) -> ObjectReference { let from_acc = RubyObjectAccess::from_objref(from); - let maybe_givtbl = from_acc.get_original_givtbl(); + let maybe_givtbl = if flags_has_fl_exivar(vm_data) { + from_acc.get_original_givtbl() + } else { + None + }; let from_start = from_acc.obj_start(); let object_size = from_acc.object_size(); let to_start = copy_context.alloc_copy(from, object_size, MIN_OBJ_ALIGN, 0, semantics); @@ -92,6 +96,7 @@ impl ObjectModel for VMObjectModel { } let to_acc = RubyObjectAccess::from_objref(to_obj); to_acc.set_has_moved_givtbl(); + warn!("{} has moved givtbl", to_obj); } to_obj From ced6160657f6a1bce53f5a32f490a552196ab935 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 31 Oct 2023 13:27:18 +0800 Subject: [PATCH 09/10] Use FL_PROMOTED bit as part of forwarding state --- mmtk/src/abi.rs | 106 ++++++++++++++++++++++++++------------- mmtk/src/object_model.rs | 20 +++++--- 2 files changed, 82 insertions(+), 44 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 003df09..5663fe3 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -17,12 +17,40 @@ const HIDDEN_SIZE_MASK: usize = 0x0000FFFFFFFFFFFF; const BUILTIN_TYPE_MASK: usize = 0x1f; const T_MOVED: usize = 0x1e; -const T_MOVED_FORWARDED: usize = 0x3e; const FORWARDING_POINTER_OFFSET: usize = 16; +const RUBY_FL_PROMOTED: usize = 1 << 5; const RUBY_FL_EXIVAR: usize = 1 << 10; +fn flags_is_forwarding_not_triggered_yet(flags: usize) -> bool { + !flags_is_forwarded_or_being_forwarded(flags) +} + +fn flags_is_being_forwarded(flags: usize) -> bool { + (flags & RUBY_FL_PROMOTED) != 0 +} + +fn flags_is_forwarded(flags: usize) -> bool { + (flags & BUILTIN_TYPE_MASK) == T_MOVED +} + +fn flags_is_forwarded_or_being_forwarded(flags: usize) -> bool { + flags_is_being_forwarded(flags) || flags_is_forwarded(flags) +} + +fn flags_set_being_forwarded(flags: usize) -> usize { + flags | RUBY_FL_PROMOTED +} + +pub fn flags_clear_being_forwarded(flags: usize) -> usize { + flags & !RUBY_FL_PROMOTED +} + +fn flags_set_forwarded(flags: usize) -> usize { + (flags_clear_being_forwarded(flags) & !BUILTIN_TYPE_MASK) | T_MOVED +} + pub fn flags_has_fl_exivar(flags: usize) -> bool { flags & RUBY_FL_EXIVAR != 0 } @@ -139,32 +167,36 @@ impl RubyObjectAccess { } } - pub fn attempt_to_forward(&self) -> Option { + pub fn flags_field_atomic(&self) -> &'static AtomicUsize { + unsafe { self.objref.to_raw_address().as_ref::() } + } + + pub fn attempt_to_forward(&self) -> bool { trace!("attempt_to_forward({})", self.objref); - let flags = unsafe { self.objref.to_raw_address().as_ref::() }; + let flags = self.flags_field_atomic(); let old_value = flags.load(Ordering::Relaxed); - if old_value & BUILTIN_TYPE_MASK != T_MOVED { + if flags_is_forwarding_not_triggered_yet(old_value) { trace!("Forwarding not triggered, yet. old value: {:x}", old_value); // Forwarding not triggered yet. Try to transition the state. - let new_value = T_MOVED; + let new_value = flags_set_being_forwarded(old_value); match flags.compare_exchange(old_value, new_value, Ordering::Relaxed, Ordering::Relaxed) { Ok(actual) => { debug_assert_eq!(actual, old_value); - Some(old_value) + true } Err(actual) => { debug_assert_ne!(actual, old_value); - debug_assert_eq!(actual & BUILTIN_TYPE_MASK, T_MOVED); - None + debug_assert!(flags_is_forwarded_or_being_forwarded(actual)); + false } } } else { trace!("Already forwarded. old value: {:x}", old_value); // Already forwarded. Fail. - None + false } } @@ -179,53 +211,55 @@ impl RubyObjectAccess { .store::(new_object) } + let flags = self.flags_field_atomic(); + // Note: no need for atomic RMW operation because we currently owns this object. + let old_value = flags.load(Ordering::Relaxed); + debug_assert!(flags_is_being_forwarded(old_value)); + let new_value = flags_set_forwarded(old_value); // Ordering matters. The following store is a release store. - - let flags = unsafe { self.objref.to_raw_address().as_ref::() }; - flags.store(T_MOVED_FORWARDED, Ordering::Release); + flags.store(new_value, Ordering::Release); trace!("Obj: {} Forwarding complete!", self.objref); } - pub fn revert_forwarding_state(&self, vm_data: usize) { - trace!("revert_forwarding_state({}, {:x})", self.objref, vm_data); - let flags = unsafe { self.objref.to_raw_address().as_ref::() }; - flags.store(vm_data, Ordering::Relaxed); + pub fn revert_forwarding_state(&self) { + trace!("revert_forwarding_state({})", self.objref); + let flags = self.flags_field_atomic(); + // Note: no need for atomic RMW operation because we currently owns this object. + let old_value = flags.load(Ordering::Relaxed); + debug_assert!(flags_is_being_forwarded(old_value)); + let new_value = flags_clear_being_forwarded(old_value); + flags.store(new_value, Ordering::Relaxed); } pub fn spin_and_get_forwarded_object(&self) -> ObjectReference { trace!("spin_and_get_forwarded_object({})", self.objref); - let flags = unsafe { self.objref.to_raw_address().as_ref::() }; + let flags = self.flags_field_atomic(); loop { let value = flags.load(Ordering::Acquire); + // Ordering matters. The load above is an acquire load. trace!("Obj: {} Spinning... Old value: {:x}", self.objref, value); - // Ordering matters. The load above is an acquire load. - - match value { - T_MOVED_FORWARDED => { - trace!("Obj: {} Complete. Loading fwd ptr...", self.objref); - // Forwarded. Load forwarding pointer. - break self.load_forwarding_pointer(); - } - T_MOVED => { - // Being forwarded. - // Keep waiting - } - _ => { - trace!("Obj: {} Reverted. Loading fwd ptr...", self.objref); - // Reverted. Return self. - break self.objref; - } + if flags_is_forwarded(value) { + trace!("Obj: {} Complete. Loading fwd ptr...", self.objref); + // Load forwarding pointer. + break self.load_forwarding_pointer(); + } else if flags_is_being_forwarded(value) { + // Being forwarded. + // Keep waiting + } else { + trace!("Obj: {} Reverted. Loading fwd ptr...", self.objref); + // Reverted. Return self. + break self.objref; } } } pub fn is_forwarded(&self) -> bool { - let flags = unsafe { self.objref.to_raw_address().as_ref::() }; + let flags = self.flags_field_atomic(); let old_value = flags.load(Ordering::Relaxed); - old_value == T_MOVED_FORWARDED + flags_is_forwarded(old_value) } pub fn load_forwarding_pointer(&self) -> ObjectReference { diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 0fcaddb..5c58f32 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,6 +1,7 @@ use std::ptr::copy_nonoverlapping; +use std::sync::atomic::Ordering; -use crate::abi::{flags_has_fl_exivar, RubyObjectAccess, MIN_OBJ_ALIGN, OBJREF_OFFSET}; +use crate::abi::{flags_has_fl_exivar, RubyObjectAccess, MIN_OBJ_ALIGN, OBJREF_OFFSET, flags_clear_being_forwarded}; use crate::{abi, Ruby}; use mmtk::util::copy::{CopySemantics, GCWorkerCopyContext}; use mmtk::util::{Address, ObjectReference}; @@ -13,7 +14,7 @@ impl VMObjectModel { } impl ObjectModel for VMObjectModel { - type VMForwardingDataType = usize; + type VMForwardingDataType = (); const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); @@ -44,10 +45,12 @@ impl ObjectModel for VMObjectModel { from: ObjectReference, semantics: CopySemantics, copy_context: &mut GCWorkerCopyContext, - vm_data: Self::VMForwardingDataType, + _vm_data: Self::VMForwardingDataType, ) -> ObjectReference { let from_acc = RubyObjectAccess::from_objref(from); - let maybe_givtbl = if flags_has_fl_exivar(vm_data) { + let flags = from_acc.flags_field_atomic(); + let old_flags = flags.load(Ordering::Relaxed); + let maybe_givtbl = if flags_has_fl_exivar(old_flags) { from_acc.get_original_givtbl() } else { None @@ -62,8 +65,9 @@ impl ObjectModel for VMObjectModel { // The `flags` field of the from-space copy is overwritten to `T_MOVED`. // Reconstruct the `flags` of the to-space copy. + let new_flags = flags_clear_being_forwarded(old_flags); unsafe { - to_payload.store::(vm_data); + to_payload.store::(new_flags); } let to_obj = ObjectReference::from_raw_address(to_payload); @@ -155,7 +159,7 @@ impl ObjectModel for VMObjectModel { } fn attempt_to_forward(object: ObjectReference) -> Option { - RubyObjectAccess::from_objref(object).attempt_to_forward() + RubyObjectAccess::from_objref(object).attempt_to_forward().then_some(()) } fn write_forwarding_state_and_forwarding_pointer( @@ -166,8 +170,8 @@ impl ObjectModel for VMObjectModel { .write_forwarding_state_and_forwarding_pointer(new_object) } - fn revert_forwarding_state(object: ObjectReference, vm_data: usize) { - RubyObjectAccess::from_objref(object).revert_forwarding_state(vm_data) + fn revert_forwarding_state(object: ObjectReference, _vm_data: Self::VMForwardingDataType) { + RubyObjectAccess::from_objref(object).revert_forwarding_state() } fn spin_and_get_forwarded_object(object: ObjectReference) -> ObjectReference { From 60ddb61db3a1cf069c1e87c26ffdcf8a45fd3d15 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 31 Oct 2023 17:28:14 +0800 Subject: [PATCH 10/10] Use another bit pattern and put fwd ptr in flags --- mmtk/src/abi.rs | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 5663fe3..52c02c3 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -18,17 +18,15 @@ const HIDDEN_SIZE_MASK: usize = 0x0000FFFFFFFFFFFF; const BUILTIN_TYPE_MASK: usize = 0x1f; const T_MOVED: usize = 0x1e; -const FORWARDING_POINTER_OFFSET: usize = 16; - const RUBY_FL_PROMOTED: usize = 1 << 5; const RUBY_FL_EXIVAR: usize = 1 << 10; fn flags_is_forwarding_not_triggered_yet(flags: usize) -> bool { - !flags_is_forwarded_or_being_forwarded(flags) + (flags & BUILTIN_TYPE_MASK) != T_MOVED && (flags & RUBY_FL_PROMOTED) == 0 } fn flags_is_being_forwarded(flags: usize) -> bool { - (flags & RUBY_FL_PROMOTED) != 0 + (flags & BUILTIN_TYPE_MASK) != T_MOVED && (flags & RUBY_FL_PROMOTED) != 0 } fn flags_is_forwarded(flags: usize) -> bool { @@ -36,7 +34,7 @@ fn flags_is_forwarded(flags: usize) -> bool { } fn flags_is_forwarded_or_being_forwarded(flags: usize) -> bool { - flags_is_being_forwarded(flags) || flags_is_forwarded(flags) + !flags_is_forwarding_not_triggered_yet(flags) } fn flags_set_being_forwarded(flags: usize) -> usize { @@ -47,8 +45,12 @@ pub fn flags_clear_being_forwarded(flags: usize) -> usize { flags & !RUBY_FL_PROMOTED } -fn flags_set_forwarded(flags: usize) -> usize { - (flags_clear_being_forwarded(flags) & !BUILTIN_TYPE_MASK) | T_MOVED +fn flags_set_forwarded_with_forwarding_pointer(forwarding_pointer: usize) -> usize { + T_MOVED | forwarding_pointer << 8 +} + +fn flags_get_forwarding_pointer(flags: usize) -> usize { + flags >> 8 } pub fn flags_has_fl_exivar(flags: usize) -> bool { @@ -206,16 +208,11 @@ impl RubyObjectAccess { self.objref, new_object ); - unsafe { - (self.objref.to_raw_address() + FORWARDING_POINTER_OFFSET) - .store::(new_object) - } let flags = self.flags_field_atomic(); // Note: no need for atomic RMW operation because we currently owns this object. - let old_value = flags.load(Ordering::Relaxed); - debug_assert!(flags_is_being_forwarded(old_value)); - let new_value = flags_set_forwarded(old_value); + let new_value = + flags_set_forwarded_with_forwarding_pointer(new_object.to_raw_address().as_usize()); // Ordering matters. The following store is a release store. flags.store(new_value, Ordering::Release); trace!("Obj: {} Forwarding complete!", self.objref); @@ -263,9 +260,11 @@ impl RubyObjectAccess { } pub fn load_forwarding_pointer(&self) -> ObjectReference { - unsafe { - (self.objref.to_raw_address() + FORWARDING_POINTER_OFFSET).load::() - } + let flags = self.flags_field_atomic(); + let old_value = flags.load(Ordering::Relaxed); + ObjectReference::from_raw_address(unsafe { + Address::from_usize(flags_get_forwarding_pointer(old_value)) + }) } }