use std::cell::{RefCell, UnsafeCell};
use std::collections::{HashMap, hash_map::Entry};
use std::ffi::c_void;
use std::marker::PhantomData;
use super::cold_init_lazy_lock::ColdInitLazyLock;
use super::dispatch::IntoDispatcher;
use std::borrow::BorrowMut;
use std::ops::ControlFlow;
use std::ptr::NonNull;
use std::rc::Rc;
use super::dispatch::Dispatcher;
use crate::any::{Any, AnyView, is_plain_inline};
use crate::error::{Error, Result, TYPE_ERROR};
use crate::function::Function;
use crate::object::{self, Object, ObjectArc};
use crate::reflection::TypeAttrColumn;
use crate::tvm_ffi_sys::{
TVMFFIAny, TVMFFIAnyViewToOwnedAny, TVMFFIFieldInfo, TVMFFIFieldSetter, TVMFFIFunctionCall,
TVMFFIGetTypeInfo, TVMFFIObject, TVMFFITypeIndex,
};
use crate::tvm_ffi_sys::{TVMFFIObjectHandle, TVMFFISEqHashKind};
use super::structural_common::{
FLAG_SEQ_HASH_IGNORE, FLAG_SETTER_IS_FUNCTION, field_def_region, for_each_field_info,
free_var_child_region, missing_field_getter_error, raw_ffi_any_into_result,
raw_ffi_any_same_as, registered_type_index, reject_error_value, result_into_raw_ffi_any,
runtime_error, seq_hash_kind, wrong_dispatcher,
};
use super::structural_visit::DefRegionKind;
use super::unchanged::is_unchanged;
use super::unchanged::{Unchanged, UnchangedOr};
use super::visit_error_context::{enrich_visit_error, update_visit_error_context};
const STRUCTURAL_MUTATE_ATTR: &str = "__s_mutate__";
const STRUCTURAL_MAYBE_INPLACE_MUTATE_ATTR: &str = "__s_maybe_inplace_mutate__";
const SHALLOW_COPY_ATTR: &str = "__ffi_shallow_copy__";
pub(super) type StructuralMutatorHandle = *mut StructuralMutatorObj;
type FStructuralMutate =
unsafe extern "C" fn(StructuralMutatorHandle, AnyView<'static>) -> TVMFFIAny;
type FStructuralVarRemapGet =
unsafe extern "C" fn(StructuralMutatorHandle, AnyView<'static>) -> TVMFFIAny;
type FStructuralVarRemapSet =
unsafe extern "C" fn(StructuralMutatorHandle, AnyView<'static>, AnyView<'static>) -> TVMFFIAny;
#[repr(C)]
struct StructuralMutatorVTable {
mutate: FStructuralMutate,
maybe_inplace_mutate: FStructuralMutate,
var_remap_get: FStructuralVarRemapGet,
var_remap_set: FStructuralVarRemapSet,
}
#[repr(C)]
pub struct StructuralMutatorObj {
base_: Object,
vtable: *const StructuralMutatorVTable,
def_region_mode: UnsafeCell<i32>,
}
impl StructuralMutatorObj {
const TYPE_KEY: &'static str = "ffi.StructuralMutator";
}
impl Default for StructuralMutatorObj {
#[inline(always)]
fn default() -> Self {
Self {
base_: Object::new(),
vtable: std::ptr::null(),
def_region_mode: UnsafeCell::new(DefRegionKind::None as i32),
}
}
}
const _: () = {
assert!(
std::mem::offset_of!(StructuralMutatorObj, vtable) == std::mem::size_of::<TVMFFIObject>()
);
assert!(
std::mem::offset_of!(StructuralMutatorObj, def_region_mode)
== std::mem::size_of::<TVMFFIObject>() + std::mem::size_of::<*const c_void>()
);
};
pub unsafe trait StructuralMutatorBase: Sized {
fn new() -> Result<Self>;
fn type_index() -> i32;
unsafe fn default_mutate_raw(runtime: *mut Self, value: MutationView<'_>) -> TVMFFIAny;
}
unsafe impl StructuralMutatorBase for StructuralMutatorObj {
#[inline(always)]
fn new() -> Result<Self> {
Ok(Self::default())
}
fn type_index() -> i32 {
static TYPE_INDEX: ColdInitLazyLock<i32> =
ColdInitLazyLock::new(|| registered_type_index(StructuralMutatorObj::TYPE_KEY));
*TYPE_INDEX
}
#[inline(always)]
unsafe fn default_mutate_raw(runtime: *mut Self, value: MutationView<'_>) -> TVMFFIAny {
let raw = value.value;
if let Some(attr) = Self::structural_mutate_hook(raw, value.mode) {
return unsafe { Self::call_structural_mutate_hook_raw(runtime, raw, attr) };
}
unsafe { Self::mutate_without_hook_raw(runtime, raw) }
}
}
#[repr(C)]
pub struct StructuralMutateEngine<Base = StructuralMutatorObj> {
base: Base,
dispatcher: *mut c_void,
remap: RefCell<StructuralVarRemap>,
}
#[derive(Default)]
pub(crate) struct StructuralVarRemap {
entries: HashMap<NonNull<TVMFFIObject>, Any>,
}
impl StructuralVarRemap {
pub(crate) fn get(&self, var: AnyView<'_>) -> Result<Option<Any>> {
let key = object_identity_key(*var.as_raw_ffi_any())?;
Ok(self.entries.get(&key).cloned())
}
pub(crate) fn set(&mut self, var: AnyView<'_>, mutated_value: &Any) -> Result<()> {
let key = object_identity_key(*var.as_raw_ffi_any())?;
let value = mutated_value.clone();
match self.entries.entry(key) {
Entry::Occupied(mut entry) => {
entry.insert(value);
}
Entry::Vacant(entry) => {
entry.insert(value);
unsafe { object::unsafe_::inc_ref(key.as_ptr()) };
}
}
Ok(())
}
}
impl Drop for StructuralVarRemap {
fn drop(&mut self) {
for key in self.entries.keys() {
unsafe { object::unsafe_::dec_ref(key.as_ptr()) };
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum InplaceMode {
#[default]
Disallow,
Allow,
}
impl InplaceMode {
#[inline(always)]
fn for_value(self, raw: TVMFFIAny) -> Self {
if self == Self::Allow && object_is_unique(raw) {
Self::Allow
} else {
Self::Disallow
}
}
}
pub struct MutationView<'a, T = Any> {
value: TVMFFIAny,
mode: InplaceMode,
#[allow(clippy::type_complexity)]
_marker: PhantomData<(AnyView<'a>, fn() -> T, Rc<()>)>,
}
impl<T> MutationView<'_, T> {
#[inline(always)]
pub(crate) fn cell(&self) -> &TVMFFIAny {
&self.value
}
}
impl<'a> MutationView<'a> {
#[inline(always)]
pub(super) fn new(value: AnyView<'a>, mode: InplaceMode) -> Self {
Self::from_raw(*value.as_raw_ffi_any(), mode)
}
#[inline(always)]
pub fn borrowed(value: AnyView<'a>) -> Self {
Self::new(value, InplaceMode::Disallow)
}
}
impl<'a> From<&'a mut Any> for MutationView<'a> {
#[inline(always)]
fn from(value: &'a mut Any) -> Self {
Self::new(AnyView::from(&*value), InplaceMode::Allow)
}
}
impl<'a, T> MutationView<'a, T> {
#[inline(always)]
fn from_raw(value: TVMFFIAny, mode: InplaceMode) -> Self {
Self {
value,
mode,
_marker: PhantomData,
}
}
#[inline(always)]
pub(crate) unsafe fn alias(&self) -> MutationView<'a, T> {
MutationView::from_raw(self.value, self.mode)
}
#[inline(always)]
pub(crate) fn reborrow(&mut self) -> MutationView<'_, T> {
MutationView::from_raw(self.value, self.mode)
}
#[inline(always)]
pub fn inplace_mode(&self) -> InplaceMode {
self.mode
}
#[inline(always)]
pub fn narrow_inplace_mode(mut self, requested: InplaceMode) -> Self {
if requested == InplaceMode::Disallow {
self.mode = InplaceMode::Disallow;
}
self
}
#[inline(always)]
pub fn as_any_view(&self) -> AnyView<'_> {
unsafe { AnyView::from_raw_ffi_any(self.value) }
}
#[inline(always)]
pub fn downcast_ref<U: crate::type_traits::DowncastRefFromAny>(&self) -> Option<&U> {
unsafe { U::downcast_ref_from_any(&self.value) }
}
#[inline(always)]
pub fn is<U: crate::type_traits::AnyCompatible>(&self) -> bool {
self.as_any_view().is::<U>()
}
#[inline(always)]
pub fn downcast<U: crate::type_traits::ContainerElement>(
self,
) -> std::result::Result<MutationView<'a, U>, Self> {
if unsafe {
U::container_check_any_strict(&self.value)
&& U::container_match_constraints(&self.value)
} {
Ok(MutationView::from_raw(self.value, self.mode))
} else {
Err(self)
}
}
#[inline(always)]
pub fn into_untyped(self) -> MutationView<'a> {
MutationView::from_raw(self.value, self.mode)
}
}
mod mutation_field {
pub trait Sealed {}
impl Sealed for crate::Any {}
impl<T: crate::object::ObjectRefCore> Sealed for T {}
}
#[doc(hidden)]
pub trait MutationField: crate::type_traits::ContainerElement + mutation_field::Sealed {}
impl MutationField for Any {}
impl<T: object::ObjectRefCore> MutationField for T {}
impl<T: object::ObjectRefCore> std::ops::Deref for MutationView<'_, T> {
type Target = T::ContainerType;
#[inline(always)]
fn deref(&self) -> &Self::Target {
unsafe { &*self.value.data_union.v_obj.cast::<T::ContainerType>() }
}
}
impl<T: object::ObjectRefCore> MutationView<'_, T> {
#[inline(always)]
pub fn field<U: MutationField>(
&mut self,
selector: impl for<'node> FnOnce(&'node T::ContainerType) -> &'node U,
) -> MutationView<'_, U> {
let node: &T::ContainerType = self;
let child = selector(node);
let offset = (child as *const U)
.addr()
.checked_sub((node as *const T::ContainerType).addr());
assert!(
offset.is_some_and(|offset| {
offset <= std::mem::size_of::<T::ContainerType>()
&& std::mem::size_of::<U>() > 0
&& std::mem::size_of::<U>() <= std::mem::size_of::<T::ContainerType>() - offset
}),
"MutationView::field must select a direct owning field"
);
let mut value = TVMFFIAny::new();
unsafe { U::container_copy_to_any_view(child, &mut value) };
let mode = self.mode.for_value(self.value);
MutationView::from_raw(value, mode)
}
}
pub struct StructuralMutatorView<'a, D, Base = StructuralMutatorObj> {
runtime: StructuralMutatorHandle,
#[allow(clippy::type_complexity)]
_marker: PhantomData<(&'a mut (), fn(&mut D) -> Base, Rc<()>)>,
}
impl<D, Base: StructuralMutatorBase> StructuralMutatorView<'_, D, Base> {
#[inline(always)]
pub fn base(&self) -> &Base {
unsafe { &*self.runtime.cast::<Base>() }
}
#[inline(always)]
fn reborrow(&self) -> StructuralMutatorView<'_, D, Base> {
StructuralMutatorView {
runtime: self.runtime,
_marker: PhantomData,
}
}
#[inline(always)]
pub fn def_region_kind(&self) -> DefRegionKind {
unsafe { std::mem::transmute(*(*self.runtime).def_region_mode.get()) }
}
#[inline(always)]
pub fn with_def_region_kind<R>(
&mut self,
kind: DefRegionKind,
callback: impl FnOnce(&mut StructuralMutatorView<'_, D, Base>) -> R,
) -> R {
StructuralMutatorObj::with_def_region_kind(self.runtime, kind, |_| {
callback(&mut self.reborrow())
})
}
#[inline(always)]
pub fn mutate<'input, T>(
&mut self,
dispatch: &mut D,
value: impl Into<MutationView<'input, T>>,
) -> Result<UnchangedOr<Any>> {
let value = value.into();
unsafe {
let context =
StructuralMutateEngine::<Base>::checked_dispatcher(self.runtime, dispatch);
(*StructuralMutateEngine::<Base>::from_handle(self.runtime)).dispatcher = context;
StructuralMutatorObj::call_mutator(self.runtime, value.value, value.mode)
}
.and_then(UnchangedOr::from_carrier)
}
#[inline(always)]
pub fn default_mutate<'input, T>(
&mut self,
dispatch: &mut D,
value: impl Into<MutationView<'input, T>>,
) -> Result<UnchangedOr<Any>> {
let value = value.into();
unsafe {
let context =
StructuralMutateEngine::<Base>::checked_dispatcher(self.runtime, dispatch);
raw_ffi_any_into_result(StructuralMutateEngine::<Base>::publish_and_descend_raw(
self.runtime,
context,
value.value,
value.mode,
))
}
.and_then(UnchangedOr::from_carrier)
}
#[inline(always)]
pub fn var_remap_get(&mut self, dispatch: &mut D, var: AnyView<'_>) -> Result<Option<Any>> {
unsafe {
StructuralMutateEngine::<Base>::checked_dispatcher(self.runtime, dispatch);
StructuralMutateEngine::<Base>::runtime_var_remap_get(
self.runtime,
*var.as_raw_ffi_any(),
)
}
}
#[inline(always)]
pub fn var_remap_set(&mut self, dispatch: &mut D, var: AnyView<'_>, value: &Any) -> Result<()> {
unsafe {
StructuralMutateEngine::<Base>::checked_dispatcher(self.runtime, dispatch);
StructuralMutateEngine::<Base>::runtime_var_remap_set(
self.runtime,
*var.as_raw_ffi_any(),
value,
)
}
}
}
pub fn structural_mutate<R, H, M, D, O>(root: R, dispatcher: H) -> Result<Any>
where
R: Into<Any>,
O: BorrowMut<D>,
for<'v, 'c, 'r> H: IntoDispatcher<
(
MutationView<'v>,
&'c mut StructuralMutatorView<'r, D, StructuralMutatorObj>,
),
Option<Result<UnchangedOr<Any>>>,
M,
Target = D,
Output = O,
>,
for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
(
MutationView<'v>,
&'c mut StructuralMutatorView<'r, D, StructuralMutatorObj>,
),
Option<Result<UnchangedOr<Any>>>,
>,
{
StructuralMutateEngine::mutate_root(root, dispatcher)
}
mod callback_result_sealed {
use super::{Any, Result};
pub trait Sealed {}
impl<T: Into<Any>> Sealed for T {}
impl<T: Into<Any>> Sealed for Result<T> {}
}
#[doc(hidden)]
pub trait IntoMutateResult: callback_result_sealed::Sealed {
fn into_mutate_result(self) -> Result<UnchangedOr<Any>>;
}
impl<T: Into<Any>> IntoMutateResult for T {
#[inline(always)]
fn into_mutate_result(self) -> Result<UnchangedOr<Any>> {
Ok(UnchangedOr::from_any(self.into()))
}
}
impl<T: Into<Any>> IntoMutateResult for Result<T> {
#[inline(always)]
fn into_mutate_result(self) -> Result<UnchangedOr<Any>> {
self.map(|value| UnchangedOr::from_any(value.into()))
}
}
impl<Base: StructuralMutatorBase> StructuralMutateEngine<Base> {
#[inline(always)]
pub fn mutate_root<R, H, M, D, O>(root: R, dispatcher: H) -> Result<Any>
where
R: Into<Any>,
O: BorrowMut<D>,
for<'v, 'c, 'r> H: IntoDispatcher<
(MutationView<'v>, &'c mut StructuralMutatorView<'r, D, Base>),
Option<Result<UnchangedOr<Any>>>,
M,
Target = D,
Output = O,
>,
for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
(MutationView<'v>, &'c mut StructuralMutatorView<'r, D, Base>),
Option<Result<UnchangedOr<Any>>>,
>,
{
let mut storage = dispatcher.into_dispatcher();
Self::run_mutator::<D, true>(root.into(), storage.borrow_mut())
}
#[inline(always)]
unsafe fn from_handle(runtime: StructuralMutatorHandle) -> *mut Self {
runtime.cast()
}
#[inline(always)]
unsafe fn checked_dispatcher<D>(
runtime: StructuralMutatorHandle,
dispatch: &mut D,
) -> *mut c_void {
let context = std::ptr::from_mut(dispatch).cast();
if context != unsafe { (*Self::from_handle(runtime)).dispatcher } {
core::hint::cold_path();
wrong_dispatcher();
}
context
}
#[inline(always)]
fn vtable<D, const CALLBACK_CONTEXT: bool>() -> &'static StructuralMutatorVTable
where
for<'s, 'v, 'c, 'r> &'s mut D: Dispatcher<
(MutationView<'v>, &'c mut StructuralMutatorView<'r, D, Base>),
Option<Result<UnchangedOr<Any>>>,
>,
{
&StructuralMutatorVTable {
mutate: Self::rust_vtable_mutate::<D, false, CALLBACK_CONTEXT>,
maybe_inplace_mutate: Self::rust_vtable_mutate::<D, true, CALLBACK_CONTEXT>,
var_remap_get: Self::rust_vtable_var_remap_get,
var_remap_set: Self::rust_vtable_var_remap_set,
}
}
unsafe extern "C" fn rust_vtable_mutate<D, const INPLACE: bool, const CALLBACK_CONTEXT: bool>(
runtime: StructuralMutatorHandle,
value: AnyView<'static>,
) -> TVMFFIAny
where
for<'s, 'v, 'c, 'r> &'s mut D: Dispatcher<
(MutationView<'v>, &'c mut StructuralMutatorView<'r, D, Base>),
Option<Result<UnchangedOr<Any>>>,
>,
{
let context = unsafe { (*Self::from_handle(runtime)).dispatcher };
let raw = *value.as_raw_ffi_any();
let mode = if INPLACE {
InplaceMode::Allow
} else {
InplaceMode::Disallow
};
let mode = mode.for_value(raw);
let dispatch = unsafe { &mut *context.cast::<D>() };
let mut mutator = StructuralMutatorView {
runtime,
_marker: PhantomData,
};
let result = match Dispatcher::dispatch_tuple(
&mut *dispatch,
(MutationView::new(value, mode), &mut mutator),
) {
Some(result) => {
let result = result.map(Any::from).and_then(reject_error_value);
result_into_raw_ffi_any(if CALLBACK_CONTEXT {
result.map_err(|error| update_visit_error_context(error, raw))
} else {
result
})
}
None => unsafe {
Self::publish_and_descend_raw(
runtime,
std::ptr::from_mut(dispatch).cast(),
raw,
mode,
)
},
};
unsafe { (*Self::from_handle(runtime)).dispatcher = context };
result
}
#[inline(always)]
unsafe fn publish_and_descend_raw(
runtime: StructuralMutatorHandle,
context: *mut c_void,
raw: TVMFFIAny,
mode: InplaceMode,
) -> TVMFFIAny {
unsafe { (*Self::from_handle(runtime)).dispatcher = context };
unsafe {
<Base as StructuralMutatorBase>::default_mutate_raw(
runtime.cast(),
MutationView::new(AnyView::from_raw_ffi_any(raw), mode),
)
}
}
#[inline(always)]
unsafe fn runtime_var_remap_get(
runtime: StructuralMutatorHandle,
raw: TVMFFIAny,
) -> Result<Option<Any>> {
unsafe {
(*Self::from_handle(runtime))
.remap
.borrow()
.get(AnyView::from_raw_ffi_any(raw))
}
}
#[inline(always)]
unsafe fn runtime_var_remap_set(
runtime: StructuralMutatorHandle,
raw: TVMFFIAny,
value: &Any,
) -> Result<()> {
unsafe {
(*Self::from_handle(runtime))
.remap
.borrow_mut()
.set(AnyView::from_raw_ffi_any(raw), value)
}
}
unsafe extern "C" fn rust_vtable_var_remap_get(
runtime: StructuralMutatorHandle,
var: AnyView<'static>,
) -> TVMFFIAny {
match unsafe { Self::runtime_var_remap_get(runtime, *var.as_raw_ffi_any()) } {
Ok(Some(value)) => Any::into_raw_ffi_any(value),
Ok(None) => TVMFFIAny::new(),
Err(error) => result_into_raw_ffi_any(Err(error)),
}
}
unsafe extern "C" fn rust_vtable_var_remap_set(
runtime: StructuralMutatorHandle,
var: AnyView<'static>,
replacement: AnyView<'static>,
) -> TVMFFIAny {
let result =
raw_ffi_any_to_owned_any(*replacement.as_raw_ffi_any()).and_then(|value| unsafe {
Self::runtime_var_remap_set(runtime, *var.as_raw_ffi_any(), &value)
});
result_into_raw_ffi_any(result.map(|()| Any::new()))
}
pub(super) fn run_mutator<D, const CALLBACK_CONTEXT: bool>(
root: Any,
dispatch: &mut D,
) -> Result<Any>
where
for<'s, 'v, 'c, 'r> &'s mut D: Dispatcher<
(MutationView<'v>, &'c mut StructuralMutatorView<'r, D, Base>),
Option<Result<UnchangedOr<Any>>>,
>,
{
let mut base = Base::new()?;
let type_index = Base::type_index();
unsafe {
let prefix = std::ptr::from_mut(&mut base).cast::<StructuralMutatorObj>();
*(*prefix).def_region_mode.get() = DefRegionKind::None as i32;
}
let mut engine = unsafe {
ObjectArc::<Object>::new_opaque_unchecked(
Self {
base,
dispatcher: std::ptr::from_mut(dispatch).cast(),
remap: RefCell::new(StructuralVarRemap::default()),
},
type_index,
)
};
let runtime = ObjectArc::as_mut_ptr(&mut engine).cast::<StructuralMutatorObj>();
unsafe { (*runtime).vtable = Self::vtable::<D, CALLBACK_CONTEXT>() };
let result =
StructuralMutatorObj::call_mutator(runtime, *root.as_raw_ffi_any(), InplaceMode::Allow);
drop(engine);
match result {
Ok(value) => Ok(if is_unchanged(&value) { root } else { value }),
Err(error) => Err(enrich_visit_error(error, *root.as_raw_ffi_any())),
}
}
}
impl StructuralMutatorObj {
#[inline(always)]
unsafe fn with_native_mutation_context_raw(result: TVMFFIAny, raw: TVMFFIAny) -> TVMFFIAny {
if result.type_index != TVMFFITypeIndex::kTVMFFIError as i32 {
result
} else {
core::hint::cold_path();
unsafe { Self::with_mutation_context_raw(result, raw) }
}
}
#[cold]
#[inline(never)]
unsafe extern "C-unwind" fn with_mutation_context_raw(
result: TVMFFIAny,
raw: TVMFFIAny,
) -> TVMFFIAny {
unsafe {
result_into_raw_ffi_any(
raw_ffi_any_into_result(result)
.map_err(|error| update_visit_error_context(error, raw)),
)
}
}
fn map_reflected(
runtime: StructuralMutatorHandle,
raw: TVMFFIAny,
def_region_kind: DefRegionKind,
type_info: *const crate::tvm_ffi_sys::TVMFFITypeInfo,
) -> Result<Any> {
let inherited_region =
free_var_child_region(def_region_kind, unsafe { seq_hash_kind(type_info) });
let output = shallow_copy(raw)?;
let output_raw = *output.as_raw_ffi_any();
let output_object = unsafe { output_raw.data_union.v_obj.cast::<u8>() };
if output_object.is_null() {
return Err(runtime_error(
"native structural mutate: shallow copy has a null object pointer",
));
}
let mut field_changed = false;
let mut failure: Option<Error> = None;
unsafe {
for_each_field_info(type_info, &mut |field| {
if field.flags & FLAG_SEQ_HASH_IGNORE != 0 {
return ControlFlow::Continue(());
}
match Self::map_reflected_field(
runtime,
output_object,
field,
inherited_region,
&mut field_changed,
) {
Ok(()) => ControlFlow::Continue(()),
Err(error) => {
failure = Some(error);
ControlFlow::Break(())
}
}
});
}
if let Some(error) = failure {
return Err(error);
}
if field_changed {
Ok(output)
} else {
Ok(Unchanged.into())
}
}
unsafe fn map_reflected_field(
runtime: StructuralMutatorHandle,
output_object: *mut u8,
field: &TVMFFIFieldInfo,
inherited_region: DefRegionKind,
field_changed: &mut bool,
) -> Result<()> {
let Some(getter) = field.getter else {
return Err(unsafe {
missing_field_getter_error("structural mutate", field.name.as_str())
});
};
let field_offset = usize::try_from(field.offset)
.map_err(|_| unsafe { invalid_field_offset_error(field.name.as_str()) })?;
let source_address = unsafe { output_object.add(field_offset).cast::<c_void>() };
let mut child = Any::new();
if unsafe { getter(source_address, Any::as_raw_ffi_any_mut(&mut child)) } != 0 {
return Err(Error::from_raised());
}
let child_raw = *child.as_raw_ffi_any();
let child_region = field_def_region(field, inherited_region);
let mapped = Self::with_def_region_kind(runtime, child_region, |_| {
Self::call_mutator(runtime, child_raw, InplaceMode::Disallow)
})?;
if is_unchanged(&mapped) || raw_ffi_any_same_as(child_raw, *mapped.as_raw_ffi_any()) {
return Ok(());
}
call_field_setter(field, source_address, mapped.as_raw_ffi_any())?;
*field_changed = true;
Ok(())
}
#[inline(always)]
fn call_mutator(
runtime: StructuralMutatorHandle,
raw: TVMFFIAny,
mode: InplaceMode,
) -> Result<Any> {
let callback = unsafe {
if mode.for_value(raw) == InplaceMode::Allow {
(*(*runtime).vtable).maybe_inplace_mutate
} else {
(*(*runtime).vtable).mutate
}
};
unsafe {
let view = AnyView::from_raw_ffi_any(raw);
raw_ffi_any_into_result(callback(runtime, view))
}
}
#[inline(always)]
fn structural_mutate_hook(raw: TVMFFIAny, mode: InplaceMode) -> Option<TVMFFIAny> {
if mode.for_value(raw) == InplaceMode::Allow
&& let Some(attr) = structural_mutate_column::<true>().get_raw_ffi_any(raw.type_index)
&& (attr.type_index == TVMFFITypeIndex::kTVMFFIOpaquePtr as i32
|| attr.type_index == TVMFFITypeIndex::kTVMFFIFunction as i32)
{
return Some(attr);
}
let attr = structural_mutate_column::<false>().get_raw_ffi_any(raw.type_index)?;
(attr.type_index != TVMFFITypeIndex::kTVMFFINone as i32).then_some(attr)
}
#[inline(always)]
pub unsafe fn call_structural_mutate_hook_raw(
runtime: *mut Self,
value: TVMFFIAny,
hook: TVMFFIAny,
) -> TVMFFIAny {
unsafe {
if hook.type_index == TVMFFITypeIndex::kTVMFFIOpaquePtr as i32 {
let callback: FStructuralMutate = std::mem::transmute(hook.data_union.v_ptr);
Self::with_native_mutation_context_raw(
callback(runtime, AnyView::from_raw_ffi_any(value)),
value,
)
} else {
Self::call_structural_mutate_hook_slow_raw(runtime, value, hook)
}
}
}
#[cold]
#[inline(never)]
unsafe extern "C-unwind" fn call_structural_mutate_hook_slow_raw(
runtime: StructuralMutatorHandle,
raw: TVMFFIAny,
attr: TVMFFIAny,
) -> TVMFFIAny {
let result = if attr.type_index == TVMFFITypeIndex::kTVMFFIFunction as i32 {
unsafe {
Function::try_from(AnyView::from_raw_ffi_any(attr)).and_then(|function| {
function.call_packed(&[
Self::borrowed_mutator_view(runtime),
AnyView::from_raw_ffi_any(raw),
])
})
}
} else {
Err(invalid_mutation_hook_error())
};
result_into_raw_ffi_any(result.map_err(|error| update_visit_error_context(error, raw)))
}
#[inline(always)]
unsafe fn borrowed_mutator_view<'a>(runtime: StructuralMutatorHandle) -> AnyView<'a> {
let object = runtime.cast::<TVMFFIObject>();
let mut raw = TVMFFIAny::new();
raw.type_index = unsafe { (*object).type_index };
raw.small_str_len = 0;
raw.data_union.v_obj = object;
unsafe { AnyView::from_raw_ffi_any(raw) }
}
#[inline(always)]
fn with_def_region_kind<T>(
mutator: StructuralMutatorHandle,
kind: DefRegionKind,
callback: impl FnOnce(DefRegionKind) -> T,
) -> T {
unsafe {
let previous = *(*mutator).def_region_mode.get();
if previous == kind as i32 || previous == DefRegionKind::Pattern as i32 {
return callback(std::mem::transmute::<i32, DefRegionKind>(previous));
}
*(*mutator).def_region_mode.get() = kind as i32;
struct Restore {
mutator: StructuralMutatorHandle,
previous: i32,
}
impl Drop for Restore {
#[inline(always)]
fn drop(&mut self) {
unsafe { *(*self.mutator).def_region_mode.get() = self.previous };
}
}
let _restore = Restore { mutator, previous };
callback(kind)
}
}
#[inline(never)]
unsafe extern "C-unwind" fn mutate_without_hook_raw(
runtime: StructuralMutatorHandle,
raw: TVMFFIAny,
) -> TVMFFIAny {
let def_region_kind =
unsafe { std::mem::transmute::<i32, DefRegionKind>(*(*runtime).def_region_mode.get()) };
result_into_raw_ffi_any(
Self::mutate_without_hook(runtime, raw, def_region_kind)
.map_err(|error| update_visit_error_context(error, raw)),
)
}
#[inline(always)]
unsafe fn var_remap_get(
runtime: StructuralMutatorHandle,
raw: TVMFFIAny,
) -> Result<Option<Any>> {
let callback = unsafe { (*(*runtime).vtable).var_remap_get };
let value =
unsafe { raw_ffi_any_into_result(callback(runtime, AnyView::from_raw_ffi_any(raw)))? };
Ok((value.type_index() != TVMFFITypeIndex::kTVMFFINone as i32).then_some(value))
}
#[inline(always)]
unsafe fn var_remap_set(
runtime: StructuralMutatorHandle,
raw: TVMFFIAny,
value: &Any,
) -> Result<()> {
let callback = unsafe { (*(*runtime).vtable).var_remap_set };
unsafe {
raw_ffi_any_into_result(callback(
runtime,
AnyView::from_raw_ffi_any(raw),
AnyView::from_raw_ffi_any(*value.as_raw_ffi_any()),
))
.map(|_| ())
}
}
#[inline(always)]
fn mutate_without_hook(
runtime: StructuralMutatorHandle,
raw: TVMFFIAny,
def_region_kind: DefRegionKind,
) -> Result<Any> {
if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
return raw_ffi_any_to_owned_any(raw);
}
let type_info = checked_type_info(raw.type_index)?;
let kind = unsafe { seq_hash_kind(type_info) };
let is_free_var = kind == TVMFFISEqHashKind::kTVMFFISEqHashKindFreeVar as i32;
let is_dag_node = kind == TVMFFISEqHashKind::kTVMFFISEqHashKindDAGNode as i32;
if is_free_var || is_dag_node {
if let Some(mutated) = unsafe { Self::var_remap_get(runtime, raw)? } {
return Ok(mutated);
}
}
if is_free_var && def_region_kind == DefRegionKind::None {
return Ok(Unchanged.into());
}
let result = Self::map_reflected(runtime, raw, def_region_kind, type_info)?;
if is_dag_node
|| (is_free_var
&& (def_region_kind == DefRegionKind::Pattern || !is_unchanged(&result)))
{
unsafe { Self::var_remap_set(runtime, raw, &result)? };
}
Ok(result)
}
}
fn shallow_copy(raw: TVMFFIAny) -> Result<Any> {
let Some(attr) = SHALLOW_COPY_COLUMN.get_raw_ffi_any(raw.type_index) else {
return Err(missing_shallow_copy_error(
raw.type_index,
SHALLOW_COPY_ATTR,
));
};
if attr.type_index != TVMFFITypeIndex::kTVMFFIFunction as i32 {
return Err(invalid_shallow_copy_hook_error(SHALLOW_COPY_ATTR));
}
let function = unsafe { attr.data_union.v_obj };
if function.is_null() {
return Err(runtime_error("shallow copy function pointer is null"));
}
let mut result = Any::new();
let status = unsafe {
TVMFFIFunctionCall(
function.cast(),
&raw,
1,
Any::as_raw_ffi_any_mut(&mut result),
)
};
if status != 0 {
return Err(Error::from_raised());
}
let result_raw = *result.as_raw_ffi_any();
let result_pointer = unsafe { result_raw.data_union.v_obj };
let source_pointer = unsafe { raw.data_union.v_obj };
if result_raw.type_index != raw.type_index
|| result_pointer.is_null()
|| result_pointer == source_pointer
{
return Err(invalid_shallow_copy_result_error());
}
Ok(result)
}
fn call_field_setter(
field: &TVMFFIFieldInfo,
field_address: *mut c_void,
value: &TVMFFIAny,
) -> Result<()> {
if field.setter.is_null() {
return Err(missing_field_setter_error(unsafe { field.name.as_str() }));
}
let return_code = unsafe {
if field.flags & FLAG_SETTER_IS_FUNCTION == 0 {
let setter: TVMFFIFieldSetter = std::mem::transmute(field.setter);
setter(field_address, value)
} else {
let mut args = [TVMFFIAny::new(), *value];
args[0].type_index = TVMFFITypeIndex::kTVMFFIOpaquePtr as i32;
args[0].data_union.v_ptr = field_address;
let mut result = Any::new();
TVMFFIFunctionCall(
field.setter as TVMFFIObjectHandle,
args.as_ptr(),
2,
Any::as_raw_ffi_any_mut(&mut result),
)
}
};
if return_code == 0 {
Ok(())
} else {
Err(Error::from_raised())
}
}
fn object_identity_key(raw: TVMFFIAny) -> Result<NonNull<TVMFFIObject>> {
if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
core::hint::cold_path();
return Err(var_remap_key_error());
}
let pointer = unsafe { raw.data_union.v_obj };
NonNull::new(pointer).ok_or_else(|| {
core::hint::cold_path();
runtime_error("native structural mutate: identity object has a null pointer")
})
}
#[inline]
fn checked_type_info(type_index: i32) -> Result<*const crate::tvm_ffi_sys::TVMFFITypeInfo> {
let info = unsafe { TVMFFIGetTypeInfo(type_index) };
if info.is_null() {
Err(unregistered_type_error(type_index))
} else {
Ok(info)
}
}
#[inline(always)]
fn object_is_unique(raw: TVMFFIAny) -> bool {
if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
return false;
}
let pointer = unsafe { raw.data_union.v_obj };
!pointer.is_null() && unsafe { object::unsafe_::is_unique(pointer) }
}
fn raw_ffi_any_to_owned_any(raw: TVMFFIAny) -> Result<Any> {
if is_plain_inline(raw.type_index) {
return Ok(unsafe { Any::from_raw_ffi_any(raw) });
}
if raw.type_index >= TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
let object = unsafe { raw.data_union.v_obj };
if object.is_null() {
return Err(runtime_error(
"native structural mutate: object-backed value has a null pointer",
));
}
unsafe { object::unsafe_::inc_ref(object) };
return Ok(unsafe { Any::from_raw_ffi_any(raw) });
}
let mut owned = Any::new();
let return_code = unsafe { TVMFFIAnyViewToOwnedAny(&raw, Any::as_raw_ffi_any_mut(&mut owned)) };
if return_code == 0 {
Ok(owned)
} else {
Err(Error::from_raised())
}
}
static STRUCTURAL_MUTATE_COLUMN: ColdInitLazyLock<TypeAttrColumn> = ColdInitLazyLock::new(|| {
TypeAttrColumn::lookup(STRUCTURAL_MUTATE_ATTR).expect("missing __s_mutate__ column")
});
static STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
TypeAttrColumn::lookup(STRUCTURAL_MAYBE_INPLACE_MUTATE_ATTR)
.expect("missing __s_maybe_inplace_mutate__ column")
});
#[inline(always)]
fn structural_mutate_column<const INPLACE: bool>() -> TypeAttrColumn {
if INPLACE {
*STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN
} else {
*STRUCTURAL_MUTATE_COLUMN
}
}
static SHALLOW_COPY_COLUMN: ColdInitLazyLock<TypeAttrColumn> = ColdInitLazyLock::new(|| {
TypeAttrColumn::lookup(SHALLOW_COPY_ATTR).expect("missing __ffi_shallow_copy__ column")
});
#[cold]
#[inline(never)]
fn missing_shallow_copy_error(type_index: i32, attribute: &str) -> Error {
Error::new(
TYPE_ERROR,
&format!(
"type `{}` cannot use reflected structural mutation because it does not define `{attribute}`",
crate::object::type_key_of_index(type_index)
),
)
}
#[cold]
#[inline(never)]
fn invalid_shallow_copy_hook_error(attribute: &str) -> Error {
Error::new(TYPE_ERROR, &format!("{attribute} must be an ffi.Function"))
}
#[cold]
#[inline(never)]
fn invalid_shallow_copy_result_error() -> Error {
Error::new(
TYPE_ERROR,
"shallow copy callback must return a distinct object with the same type as its input",
)
}
#[cold]
#[inline(never)]
fn missing_field_setter_error(name: &str) -> Error {
Error::new(
TYPE_ERROR,
&format!("cannot structurally mutate field `{name}` because it does not define a setter"),
)
}
#[cold]
#[inline(never)]
fn invalid_field_offset_error(name: &str) -> Error {
runtime_error(&format!(
"native structural mutate: reflected field `{name}` has an invalid offset"
))
}
#[cold]
#[inline(never)]
fn invalid_mutation_hook_error() -> Error {
Error::new(
TYPE_ERROR,
"__s_mutate__ must be an opaque function pointer or ffi.Function",
)
}
#[cold]
#[inline(never)]
fn var_remap_key_error() -> Error {
Error::new(
TYPE_ERROR,
"variable-remap keys must be object-backed values",
)
}
#[cold]
#[inline(never)]
fn unregistered_type_error(type_index: i32) -> Error {
runtime_error(&format!(
"native structural mutate: unregistered type index {type_index}"
))
}