use crate::runtime::vm::{FuncRefTableId, GcHeap, GcStore, I31, SendSyncPtr};
use crate::store::AutoAssertNoGc;
use crate::{
AnyRef, ExnRef, ExternRef, Func, HeapType, Result, StorageType, Val, ValType, bail_bug,
};
use core::fmt;
use core::marker;
use core::num::NonZeroU32;
use wasmtime_core::truncate::{truncate_i32_to_i8, truncate_i32_to_i16};
use wasmtime_environ::packed_option::ReservedValue;
use wasmtime_environ::{VMGcKind, VMSharedTypeIndex};
#[repr(C, align(8))]
#[derive(Debug, Clone, Copy)]
pub struct VMGcHeader {
kind: u32,
ty: VMSharedTypeIndex,
}
unsafe impl GcHeapObject for VMGcHeader {
#[inline]
fn is(_: &VMGcHeader) -> bool {
true
}
}
const _: () = {
use core::mem::offset_of;
use wasmtime_environ::*;
assert!((VM_GC_HEADER_SIZE as usize) == core::mem::size_of::<VMGcHeader>());
assert!((VM_GC_HEADER_ALIGN as usize) == core::mem::align_of::<VMGcHeader>());
assert!((VM_GC_HEADER_KIND_OFFSET as usize) == offset_of!(VMGcHeader, kind));
assert!((VM_GC_HEADER_TYPE_INDEX_OFFSET as usize) == offset_of!(VMGcHeader, ty));
};
impl VMGcHeader {
pub fn externref() -> Self {
Self::from_kind_and_index(VMGcKind::ExternRef, VMSharedTypeIndex::reserved_value())
}
pub fn from_kind_and_index(kind: VMGcKind, ty: VMSharedTypeIndex) -> Self {
let kind = kind.as_u32();
Self { kind, ty }
}
pub fn kind(&self) -> VMGcKind {
VMGcKind::from_high_bits_of_u32(self.kind)
}
pub fn reserved_u26(&self) -> u32 {
self.kind & VMGcKind::UNUSED_MASK
}
pub fn set_reserved_u26(&mut self, value: u32) {
assert!(
VMGcKind::value_fits_in_unused_bits(value),
"VMGcHeader::set_reserved_u26 with value using more than 26 bits: \
{value:#034b} ({value}, {value:#010x})"
);
self.kind &= VMGcKind::MASK;
self.kind |= value;
}
pub unsafe fn unchecked_set_reserved_u26(&mut self, value: u32) {
debug_assert_eq!(value & VMGcKind::MASK, 0);
self.kind &= VMGcKind::MASK;
self.kind |= value;
}
pub fn ty(&self) -> Option<VMSharedTypeIndex> {
if self.ty.is_reserved_value() {
None
} else {
Some(self.ty)
}
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
#[repr(transparent)]
pub struct VMGcRef(NonZeroU32);
impl<T> From<TypedGcRef<T>> for VMGcRef {
#[inline]
fn from(value: TypedGcRef<T>) -> Self {
value.gc_ref
}
}
impl fmt::LowerHex for VMGcRef {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
impl fmt::UpperHex for VMGcRef {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
impl fmt::Pointer for VMGcRef {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{self:#x}")
}
}
impl VMGcRef {
pub const I31_REF_DISCRIMINANT: u32 = 1;
pub fn from_raw_u32(raw: u32) -> Option<Self> {
Some(Self::from_raw_non_zero_u32(NonZeroU32::new(raw)?))
}
pub fn from_heap_index(index: NonZeroU32) -> Option<Self> {
if (index.get() & Self::I31_REF_DISCRIMINANT) == 0 {
Some(Self::from_raw_non_zero_u32(index))
} else {
None
}
}
pub fn from_raw_non_zero_u32(raw: NonZeroU32) -> Self {
VMGcRef(raw)
}
#[inline]
pub fn from_i31(val: I31) -> Self {
let val = (val.get_u32() << 1) | Self::I31_REF_DISCRIMINANT;
debug_assert_ne!(val, 0);
let non_zero = unsafe { NonZeroU32::new_unchecked(val) };
VMGcRef::from_raw_non_zero_u32(non_zero)
}
pub fn unchecked_copy(&self) -> Self {
VMGcRef(self.0)
}
pub fn copy_i31(&self) -> Self {
assert!(self.is_i31());
self.unchecked_copy()
}
pub fn heap_index(&self) -> Result<NonZeroU32> {
if self.is_i31() {
bail_bug!("attempted to get heap index of i31")
}
Ok(self.0)
}
pub fn as_raw_non_zero_u32(&self) -> NonZeroU32 {
self.0
}
pub fn as_raw_u32(&self) -> u32 {
self.0.get()
}
pub fn into_typed<T>(self, gc_heap: &impl GcHeap) -> Result<TypedGcRef<T>, Self>
where
T: GcHeapObject,
{
if self.is_i31() {
return Err(self);
}
if self.is_typed::<T>(gc_heap) {
Ok(TypedGcRef {
gc_ref: self,
_phantom: marker::PhantomData,
})
} else {
Err(self)
}
}
pub fn into_typed_unchecked<T>(self) -> TypedGcRef<T>
where
T: GcHeapObject,
{
debug_assert!(!self.is_i31());
TypedGcRef {
gc_ref: self,
_phantom: marker::PhantomData,
}
}
pub fn is_typed<T>(&self, gc_heap: &impl GcHeap) -> bool
where
T: GcHeapObject,
{
if self.is_i31() {
return false;
}
match gc_heap.header(&self) {
Ok(header) => T::is(header),
Err(_) => false,
}
}
#[inline]
pub fn as_typed<T>(&self, gc_heap: &impl GcHeap) -> Option<&TypedGcRef<T>>
where
T: GcHeapObject,
{
if self.is_i31() {
return None;
}
if self.is_typed::<T>(gc_heap) {
let ptr = self as *const VMGcRef;
let ret = unsafe { &*ptr.cast() };
assert!(matches!(
ret,
TypedGcRef {
gc_ref: VMGcRef(_),
_phantom
}
));
Some(ret)
} else {
None
}
}
pub fn as_typed_unchecked<T>(&self) -> &TypedGcRef<T>
where
T: GcHeapObject,
{
debug_assert!(!self.is_i31());
let ptr = self as *const VMGcRef;
let ret = unsafe { &*ptr.cast() };
assert!(matches!(
ret,
TypedGcRef {
gc_ref: VMGcRef(_),
_phantom
}
));
ret
}
pub fn gc_header<'a>(&self, gc_heap: &'a (impl GcHeap + ?Sized)) -> Option<&'a VMGcHeader> {
if self.is_i31() {
None
} else {
gc_heap.header(self).ok()
}
}
#[inline]
pub fn is_i31(&self) -> bool {
let val = self.0.get();
(val & Self::I31_REF_DISCRIMINANT) != 0
}
#[inline]
pub fn as_i31(&self) -> Option<I31> {
if self.is_i31() {
let val = self.0.get();
Some(I31::wrapping_u32(val >> 1))
} else {
None
}
}
#[inline]
pub fn unwrap_i31(&self) -> I31 {
self.as_i31().unwrap()
}
#[inline]
pub fn is_extern_ref(&self, gc_heap: &(impl GcHeap + ?Sized)) -> bool {
self.gc_header(gc_heap)
.map_or(false, |h| h.kind().matches(VMGcKind::ExternRef))
}
#[inline]
pub fn is_any_ref(&self, gc_heap: &(impl GcHeap + ?Sized)) -> bool {
self.is_i31()
|| self
.gc_header(gc_heap)
.map_or(false, |h| h.kind().matches(VMGcKind::AnyRef))
}
pub fn read_val(
&self,
store: &mut AutoAssertNoGc,
ty: &StorageType,
offset: u32,
) -> Result<Val> {
let data = store.require_gc_store_mut()?.gc_object_data(self)?;
Ok(match ty {
StorageType::I8 => Val::I32(data.read_u8(offset)?.into()),
StorageType::I16 => Val::I32(data.read_u16(offset)?.into()),
StorageType::ValType(ValType::I32) => Val::I32(data.read_i32(offset)?),
StorageType::ValType(ValType::I64) => Val::I64(data.read_i64(offset)?),
StorageType::ValType(ValType::F32) => Val::F32(data.read_u32(offset)?),
StorageType::ValType(ValType::F64) => Val::F64(data.read_u64(offset)?),
StorageType::ValType(ValType::V128) => Val::V128(data.read_v128(offset)?),
StorageType::ValType(ValType::Ref(r)) => match r.heap_type().top() {
HeapType::Extern => {
let raw = data.read_u32(offset)?;
Val::ExternRef(ExternRef::_from_raw(store, raw))
}
HeapType::Any => {
let raw = data.read_u32(offset)?;
Val::AnyRef(AnyRef::_from_raw(store, raw))
}
HeapType::Exn => {
let raw = data.read_u32(offset)?;
Val::ExnRef(ExnRef::_from_raw(store, raw))
}
HeapType::Func => {
let func_ref_id = data.read_u32(offset)?;
let func_ref_id = FuncRefTableId::from_raw(func_ref_id);
let func_ref = store
.unwrap_gc_store()
.func_ref_table
.get_untyped(func_ref_id)?;
Val::FuncRef(unsafe {
func_ref.map(|p| Func::from_vm_func_ref(store.id(), p.as_non_null()))
})
}
otherwise => bail_bug!("not a top type: {otherwise:?}"),
},
})
}
pub fn initialize_val(
&self,
store: &mut AutoAssertNoGc,
ty: &StorageType,
offset: u32,
val: Val,
) -> Result<()> {
let gcstore = store.require_gc_store_mut()?;
match val {
Val::I32(i) if ty.is_i8() => gcstore
.gc_object_data(self)?
.write_i8(offset, truncate_i32_to_i8(i)),
Val::I32(i) if ty.is_i16() => gcstore
.gc_object_data(self)?
.write_i16(offset, truncate_i32_to_i16(i)),
Val::I32(i) => gcstore.gc_object_data(self)?.write_i32(offset, i),
Val::I64(i) => gcstore.gc_object_data(self)?.write_i64(offset, i),
Val::F32(f) => gcstore.gc_object_data(self)?.write_u32(offset, f),
Val::F64(f) => gcstore.gc_object_data(self)?.write_u64(offset, f),
Val::V128(v) => gcstore.gc_object_data(self)?.write_v128(offset, v),
Val::ExternRef(x) => {
let x = match x {
None => 0,
Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
};
store
.require_gc_store_mut()?
.gc_object_data(self)?
.write_u32(offset, x)
}
Val::AnyRef(x) => {
let x = match x {
None => 0,
Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
};
store
.require_gc_store_mut()?
.gc_object_data(self)?
.write_u32(offset, x)
}
Val::ExnRef(x) => {
let x = match x {
None => 0,
Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
};
store
.require_gc_store_mut()?
.gc_object_data(self)?
.write_u32(offset, x)
}
Val::FuncRef(f) => {
let f = f.map(|f| SendSyncPtr::new(f.vm_func_ref(store)));
let gcstore = store.require_gc_store_mut()?;
let id = unsafe { gcstore.func_ref_table.intern(f) };
gcstore
.gc_object_data(self)?
.write_u32(offset, id.into_raw())
}
Val::ContRef(_) => {
bail_bug!(
"initializing continuation references in struct fields not yet supported"
);
}
}
}
pub fn write_val(
&self,
store: &mut AutoAssertNoGc,
ty: &StorageType,
offset: u32,
val: Val,
) -> Result<()> {
let gcstore = store.require_gc_store_mut()?;
let data = gcstore.gc_object_data(self)?;
match val {
Val::I32(i) if ty.is_i8() => data.write_i8(offset, truncate_i32_to_i8(i)),
Val::I32(i) if ty.is_i16() => data.write_i16(offset, truncate_i32_to_i16(i)),
Val::I32(i) => data.write_i32(offset, i),
Val::I64(i) => data.write_i64(offset, i),
Val::F32(f) => data.write_u32(offset, f),
Val::F64(f) => data.write_u64(offset, f),
Val::V128(v) => data.write_v128(offset, v),
Val::ExternRef(e) => {
let raw = data.read_u32(offset)?;
let mut gc_ref = VMGcRef::from_raw_u32(raw);
let e = match e {
Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()),
None => None,
};
let store = store.require_gc_store_mut()?;
store.write_gc_ref(&mut gc_ref, e.as_ref())?;
let data = store.gc_object_data(self)?;
data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()))
}
Val::AnyRef(a) => {
let raw = data.read_u32(offset)?;
let mut gc_ref = VMGcRef::from_raw_u32(raw);
let a = match a {
Some(a) => Some(a.try_gc_ref(store)?.unchecked_copy()),
None => None,
};
let store = store.require_gc_store_mut()?;
store.write_gc_ref(&mut gc_ref, a.as_ref())?;
let data = store.gc_object_data(self)?;
data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()))
}
Val::ExnRef(e) => {
let raw = data.read_u32(offset)?;
let mut gc_ref = VMGcRef::from_raw_u32(raw);
let e = match e {
Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()),
None => None,
};
let store = store.require_gc_store_mut()?;
store.write_gc_ref(&mut gc_ref, e.as_ref())?;
let data = store.gc_object_data(self)?;
data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()))
}
Val::FuncRef(f) => {
let f = f.map(|f| SendSyncPtr::new(f.vm_func_ref(store)));
let gcstore = store.require_gc_store_mut()?;
let id = unsafe { gcstore.func_ref_table.intern(f) };
gcstore
.gc_object_data(self)?
.write_u32(offset, id.into_raw())
}
Val::ContRef(_) => {
bail_bug!("setting continuation references in struct fields not yet supported");
}
}
}
}
pub unsafe trait GcHeapObject: Send + Sync {
fn is(header: &VMGcHeader) -> bool;
}
#[derive(Debug, PartialEq, Eq, Hash)]
#[repr(transparent)]
pub struct TypedGcRef<T> {
gc_ref: VMGcRef,
_phantom: marker::PhantomData<*mut T>,
}
impl<T> TypedGcRef<T>
where
T: GcHeapObject,
{
pub fn clone(&self, gc_store: &mut GcStore) -> Self {
Self {
gc_ref: gc_store.clone_gc_ref(&self.gc_ref),
_phantom: marker::PhantomData,
}
}
pub fn drop(self, gc_store: &mut GcStore) {
gc_store.drop_gc_ref(self.gc_ref);
}
pub fn unchecked_copy(&self) -> Self {
Self {
gc_ref: self.gc_ref.unchecked_copy(),
_phantom: marker::PhantomData,
}
}
}
impl<T> TypedGcRef<T> {
pub fn as_untyped(&self) -> &VMGcRef {
&self.gc_ref
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reserved_bits() {
let kind = VMGcKind::StructRef;
let ty = VMSharedTypeIndex::new(1234);
let mut header = VMGcHeader::from_kind_and_index(kind, ty);
assert_eq!(header.reserved_u26(), 0);
assert_eq!(header.kind(), kind);
assert_eq!(header.ty(), Some(ty));
header.set_reserved_u26(36);
assert_eq!(header.reserved_u26(), 36);
assert_eq!(header.kind(), kind);
assert_eq!(header.ty(), Some(ty));
let max = (1 << 26) - 1;
header.set_reserved_u26(max);
assert_eq!(header.reserved_u26(), max);
assert_eq!(header.kind(), kind);
assert_eq!(header.ty(), Some(ty));
header.set_reserved_u26(0);
assert_eq!(header.reserved_u26(), 0);
assert_eq!(header.kind(), kind);
assert_eq!(header.ty(), Some(ty));
let result = std::panic::catch_unwind(move || header.set_reserved_u26(max + 1));
assert!(result.is_err());
}
}