use std::cell::UnsafeCell;
use std::marker::PhantomData;
use super::dispatch::IntoDispatcher;
use std::borrow::BorrowMut;
use std::ops::ControlFlow;
use std::os::raw::c_void;
use std::rc::Rc;
use super::cold_init_lazy_lock::ColdInitLazyLock;
use crate::any::{Any, AnyView};
use crate::derive::{Object, ObjectRef};
use crate::error::{Error, Result, TYPE_ERROR};
use crate::function::Function;
use crate::object::{Object, ObjectArc};
use crate::reflection::TypeAttrColumn;
use crate::tvm_ffi_sys::{
TVMFFIAny, TVMFFIDefRegionKind, TVMFFIFieldInfo, TVMFFIGetTypeInfo, TVMFFIObject,
TVMFFITypeIndex,
};
use super::structural_common::{
FLAG_SEQ_HASH_IGNORE, field_def_region, for_each_field_info, free_var_child_region,
missing_field_getter_error, registered_type_index, runtime_error, seq_hash_kind,
wrong_dispatcher,
};
use super::visit_error_context::{enrich_visit_error, update_visit_error_context};
const STRUCTURAL_VISIT_ATTR: &str = "__s_visit__";
type StructuralVisitorHandle = *mut StructuralVisitorObj;
type FStructuralVisit =
unsafe extern "C" fn(StructuralVisitorHandle, AnyView<'static>) -> TVMFFIAny;
#[repr(C)]
struct StructuralVisitorVTable {
visit: FStructuralVisit,
}
#[repr(C)]
pub struct StructuralVisitorObj {
base_: Object,
vtable: *const StructuralVisitorVTable,
def_region_mode: UnsafeCell<i32>,
}
impl StructuralVisitorObj {
const TYPE_KEY: &'static str = "ffi.StructuralVisitor";
}
impl Default for StructuralVisitorObj {
#[inline(always)]
fn default() -> Self {
Self {
base_: Object::new(),
vtable: std::ptr::null(),
def_region_mode: UnsafeCell::new(DefRegionKind::None as i32),
}
}
}
#[repr(C)]
#[derive(Object)]
#[object(
type_key = "ffi.VisitInterrupt",
unsafe type_index = TVMFFITypeIndex::kTVMFFIVisitInterrupt,
final
)]
pub struct VisitInterruptObj {
base_: Object,
pub value: Any,
}
const _: () = {
assert!(
std::mem::offset_of!(StructuralVisitorObj, vtable) == std::mem::size_of::<TVMFFIObject>()
);
assert!(
std::mem::offset_of!(StructuralVisitorObj, def_region_mode)
== std::mem::size_of::<TVMFFIObject>() + std::mem::size_of::<*const c_void>()
);
assert!(std::mem::offset_of!(VisitInterruptObj, value) == std::mem::size_of::<TVMFFIObject>());
};
#[repr(i32)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum DefRegionKind {
#[default]
None = 0,
Pattern = 1,
Simple = 2,
}
const _: () = {
assert!(DefRegionKind::None as i32 == TVMFFIDefRegionKind::kTVMFFIDefRegionKindNone as i32);
assert!(
DefRegionKind::Pattern as i32 == TVMFFIDefRegionKind::kTVMFFIDefRegionKindPattern as i32
);
assert!(DefRegionKind::Simple as i32 == TVMFFIDefRegionKind::kTVMFFIDefRegionKindSimple as i32);
};
#[repr(C)]
#[derive(Clone, ObjectRef)]
pub struct VisitInterrupt {
data: ObjectArc<VisitInterruptObj>,
}
impl VisitInterrupt {
#[inline(always)]
pub fn new<T: Into<Any>>(payload: T) -> Self {
Self::from_object(VisitInterruptObj {
base_: Object::new(),
value: payload.into(),
})
}
}
pub unsafe trait StructuralVisitorBase: Sized {
fn new() -> Result<Self>;
fn type_index() -> i32;
unsafe fn default_visit_raw(runtime: *mut Self, value: AnyView<'_>) -> TVMFFIAny;
}
unsafe impl StructuralVisitorBase for StructuralVisitorObj {
#[inline(always)]
fn new() -> Result<Self> {
Ok(Self::default())
}
fn type_index() -> i32 {
static TYPE_INDEX: ColdInitLazyLock<i32> =
ColdInitLazyLock::new(|| registered_type_index(StructuralVisitorObj::TYPE_KEY));
*TYPE_INDEX
}
#[inline(always)]
unsafe fn default_visit_raw(runtime: *mut Self, value: AnyView<'_>) -> TVMFFIAny {
unsafe { Self::visit_children_raw(runtime, *value.as_raw_ffi_any()) }
}
}
#[repr(C)]
pub struct StructuralVisitEngine<Base = StructuralVisitorObj> {
base: Base,
dispatcher: *mut c_void,
}
pub struct StructuralVisitorView<'a, D, Base = StructuralVisitorObj> {
runtime: StructuralVisitorHandle,
#[allow(clippy::type_complexity)]
_marker: PhantomData<(&'a mut (), fn(&mut D) -> Base, Rc<()>)>,
}
impl<D, Base: StructuralVisitorBase> StructuralVisitorView<'_, D, Base> {
#[inline(always)]
pub fn base(&self) -> &Base {
unsafe { &*self.runtime.cast::<Base>() }
}
#[inline(always)]
pub fn def_region_kind(&self) -> DefRegionKind {
unsafe { std::mem::transmute(*(*self.runtime).def_region_mode.get()) }
}
#[inline(always)]
pub fn with_def_region_kind<R>(
&mut self,
kind: DefRegionKind,
callback: impl FnOnce(&mut Self) -> R,
) -> R {
StructuralVisitorObj::with_def_region_kind(self.runtime, kind, || callback(self))
}
#[inline(always)]
pub fn visit<'input, T>(&mut self, dispatch: &mut D, value: T) -> Result<Option<VisitInterrupt>>
where
T: Into<AnyView<'input>>,
{
let value: AnyView<'input> = value.into();
unsafe {
visit_result_from_any(Any::from_raw_ffi_any(
StructuralVisitEngine::<Base>::visit_raw(
self.runtime,
dispatch,
*value.as_raw_ffi_any(),
),
))
}
}
#[inline(always)]
pub fn default_visit<'input, T>(
&self,
dispatch: &mut D,
value: T,
) -> Result<Option<VisitInterrupt>>
where
T: Into<AnyView<'input>>,
{
unsafe {
let context = StructuralVisitEngine::<Base>::checked_dispatcher(self.runtime, dispatch);
let value: AnyView<'input> = value.into();
visit_result_from_any(Any::from_raw_ffi_any(
StructuralVisitEngine::<Base>::default_visit_raw(
self.runtime,
context,
*value.as_raw_ffi_any(),
),
))
}
}
}
pub fn structural_visit<'input, R, H, M, D, O>(
root: R,
dispatcher: H,
) -> Result<Option<VisitInterrupt>>
where
R: Into<AnyView<'input>>,
O: BorrowMut<D>,
for<'v, 'c, 'r> H: IntoDispatcher<
(
AnyView<'v>,
&'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
),
Option<Result<Option<VisitInterrupt>>>,
M,
Target = D,
Output = O,
>,
for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
(
AnyView<'v>,
&'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
),
Option<Result<Option<VisitInterrupt>>>,
>,
{
StructuralVisitEngine::visit_root(root, dispatcher)
}
impl super::dispatch::CallbackResult<Result<Option<VisitInterrupt>>>
for Result<Option<VisitInterrupt>>
{
#[inline(always)]
fn callback_result(self) -> Result<Option<VisitInterrupt>> {
self
}
}
impl<Base: StructuralVisitorBase> StructuralVisitEngine<Base> {
#[inline(always)]
pub fn visit_root<'input, R, H, M, D, O>(
root: R,
dispatcher: H,
) -> Result<Option<VisitInterrupt>>
where
R: Into<AnyView<'input>>,
O: BorrowMut<D>,
for<'v, 'c, 'r> H: IntoDispatcher<
(AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
Option<Result<Option<VisitInterrupt>>>,
M,
Target = D,
Output = O,
>,
for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
(AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
Option<Result<Option<VisitInterrupt>>>,
>,
{
let mut storage = dispatcher.into_dispatcher();
let root = root.into();
Self::run_visitor::<D>(*root.as_raw_ffi_any(), storage.borrow_mut())
}
#[inline(always)]
unsafe fn from_handle(runtime: StructuralVisitorHandle) -> *mut Self {
runtime.cast()
}
#[inline(always)]
unsafe fn visit_raw<D>(
runtime: StructuralVisitorHandle,
dispatch: &mut D,
raw: TVMFFIAny,
) -> TVMFFIAny {
let context = unsafe { Self::checked_dispatcher(runtime, dispatch) };
unsafe { (*Self::from_handle(runtime)).dispatcher = context };
StructuralVisitorObj::call_visitor_raw(runtime, raw)
}
#[inline(always)]
unsafe fn checked_dispatcher<D>(
runtime: StructuralVisitorHandle,
dispatch: &mut D,
) -> *mut c_void {
let current = dispatcher_ptr(dispatch);
if current != unsafe { (*Self::from_handle(runtime)).dispatcher } {
core::hint::cold_path();
wrong_dispatcher();
}
current
}
#[inline(always)]
fn vtable<D>() -> &'static StructuralVisitorVTable
where
for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
(AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
Option<Result<Option<VisitInterrupt>>>,
>,
{
&StructuralVisitorVTable {
visit: Self::rust_vtable_visit_raw::<D>,
}
}
unsafe extern "C" fn rust_vtable_visit_raw<D>(
runtime: StructuralVisitorHandle,
value: AnyView<'static>,
) -> TVMFFIAny
where
for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
(AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
Option<Result<Option<VisitInterrupt>>>,
>,
{
let context = unsafe { (*Self::from_handle(runtime)).dispatcher };
let raw = *value.as_raw_ffi_any();
let result = if raw.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
core::hint::cold_path();
TVMFFIAny::new()
} else {
let dispatch = unsafe { &mut *context.cast::<D>() };
let mut visitor = unsafe { Self::visitor_context::<D>(runtime) };
match unsafe {
crate::Dispatcher::dispatch_tuple(
&mut *dispatch,
(AnyView::from_raw_ffi_any(raw), &mut visitor),
)
} {
Some(result) => visit_result_raw(result, raw),
None => unsafe { Self::default_visit_raw(runtime, dispatcher_ptr(dispatch), raw) },
}
};
unsafe { (*Self::from_handle(runtime)).dispatcher = context };
result
}
#[inline(always)]
unsafe fn visitor_context<'a, D>(
runtime: StructuralVisitorHandle,
) -> StructuralVisitorView<'a, D, Base> {
StructuralVisitorView {
runtime,
_marker: PhantomData,
}
}
#[inline(always)]
unsafe fn default_visit_raw(
runtime: StructuralVisitorHandle,
context: *mut c_void,
raw: TVMFFIAny,
) -> TVMFFIAny {
unsafe { (*Self::from_handle(runtime)).dispatcher = context };
unsafe {
StructuralVisitorObj::with_visit_context_raw(
Base::default_visit_raw(runtime.cast(), AnyView::from_raw_ffi_any(raw)),
raw,
)
}
}
pub(super) fn run_visitor<D>(
root: TVMFFIAny,
dispatch: &mut D,
) -> Result<Option<VisitInterrupt>>
where
for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
(AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
Option<Result<Option<VisitInterrupt>>>,
>,
{
let base = Base::new()?;
let type_index = Base::type_index();
let mut engine = unsafe {
ObjectArc::<Object>::new_opaque_unchecked(
StructuralVisitEngine {
base,
dispatcher: dispatcher_ptr(dispatch),
},
type_index,
)
};
let runtime = ObjectArc::as_mut_ptr(&mut engine).cast::<StructuralVisitorObj>();
unsafe {
(*runtime).vtable = Self::vtable::<D>();
*(*runtime).def_region_mode.get() = DefRegionKind::None as i32;
}
let result =
unsafe { Any::from_raw_ffi_any(StructuralVisitorObj::call_visitor_raw(runtime, root)) };
drop(engine);
visit_result_from_any(result).map_err(|error| enrich_visit_error(error, root))
}
}
impl StructuralVisitorObj {
#[inline(always)]
fn with_visit_context_raw(result: TVMFFIAny, value: TVMFFIAny) -> TVMFFIAny {
if result.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
result
} else {
Self::with_visit_stop_context_raw(result, value)
}
}
#[cold]
#[inline(never)]
extern "C-unwind" fn with_visit_stop_context_raw(
result: TVMFFIAny,
value: TVMFFIAny,
) -> TVMFFIAny {
if result.type_index != TVMFFITypeIndex::kTVMFFIError as i32
|| value.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32
{
result
} else {
unsafe {
let error = take_error(Any::from_raw_ffi_any(result));
Any::into_raw_ffi_any(update_visit_error_context(error, value).into())
}
}
}
#[inline(always)]
unsafe fn visit_children_raw(runtime: StructuralVisitorHandle, value: TVMFFIAny) -> TVMFFIAny {
if let Some(attr) = structural_visit_column().get_raw_ffi_any(value.type_index)
&& attr.type_index != TVMFFITypeIndex::kTVMFFINone as i32
{
return unsafe { Self::call_structural_visit_hook_raw(runtime, value, attr) };
}
if value.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
TVMFFIAny::new()
} else {
unsafe { Self::visit_reflected_fields_raw(runtime, value) }
}
}
#[inline(never)]
unsafe extern "C-unwind" fn visit_reflected_fields_raw(
runtime: StructuralVisitorHandle,
value: TVMFFIAny,
) -> TVMFFIAny {
let type_info = unsafe { TVMFFIGetTypeInfo(value.type_index) };
if type_info.is_null() {
return Any::into_raw_ffi_any(unregistered_visit_type_error(value.type_index).into());
}
let def_region_kind = unsafe {
free_var_child_region(
std::mem::transmute::<i32, DefRegionKind>(*(*runtime).def_region_mode.get()),
seq_hash_kind(type_info),
)
};
let object = unsafe { value.data_union.v_obj } as *mut u8;
let halted = unsafe {
for_each_field_info(type_info, &mut |field| {
let result = Self::visit_reflected_field(runtime, object, field, def_region_kind);
if result.is_none() {
ControlFlow::Continue(())
} else {
ControlFlow::Break(result)
}
})
};
Any::into_raw_ffi_any(halted.unwrap_or_else(Any::new))
}
unsafe fn visit_reflected_field(
runtime: StructuralVisitorHandle,
object: *mut u8,
field: &TVMFFIFieldInfo,
inherited_region: DefRegionKind,
) -> Any {
if field.flags & FLAG_SEQ_HASH_IGNORE != 0 {
return Any::new();
}
let Some(getter) = field.getter else {
return Any::from(unsafe {
missing_field_getter_error("visitor", field.name.as_str())
});
};
let address = unsafe { object.offset(field.offset as isize) as *mut c_void };
let mut child = Any::new();
if unsafe { getter(address, Any::as_raw_ffi_any_mut(&mut child)) } != 0 {
return Any::from(Error::from_raised());
}
let borrowed = AnyView::from(&child);
let child_region = field_def_region(field, inherited_region);
Self::with_def_region_kind(runtime, child_region, || unsafe {
Any::from_raw_ffi_any(Self::call_visitor_raw(runtime, *borrowed.as_raw_ffi_any()))
})
}
#[inline(always)]
fn call_visitor_raw(runtime: StructuralVisitorHandle, value: TVMFFIAny) -> TVMFFIAny {
if value.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
core::hint::cold_path();
return TVMFFIAny::new();
}
unsafe {
let callback = (*(*runtime).vtable).visit;
callback(runtime, AnyView::from_raw_ffi_any(value))
}
}
#[inline(always)]
pub unsafe fn call_structural_visit_hook_raw(
runtime: *mut Self,
value: TVMFFIAny,
hook: TVMFFIAny,
) -> TVMFFIAny {
unsafe {
if hook.type_index == TVMFFITypeIndex::kTVMFFIOpaquePtr as i32 {
let pointer = hook.data_union.v_ptr;
let hook: FStructuralVisit = std::mem::transmute(pointer);
hook(runtime, AnyView::from_raw_ffi_any(value))
} else {
core::hint::cold_path();
Self::call_structural_visit_hook_slow_raw(runtime, value, hook)
}
}
}
#[cold]
#[inline(never)]
extern "C-unwind" fn call_structural_visit_hook_slow_raw(
runtime: StructuralVisitorHandle,
value: TVMFFIAny,
hook: TVMFFIAny,
) -> TVMFFIAny {
let result = unsafe {
Function::try_from(AnyView::from_raw_ffi_any(hook)).and_then(|function| {
let object = runtime.cast::<TVMFFIObject>();
let mut runtime = TVMFFIAny::new();
runtime.type_index = (*object).type_index;
runtime.data_union.v_obj = object;
function.call_packed(&[
AnyView::from_raw_ffi_any(runtime),
AnyView::from_raw_ffi_any(value),
])
})
};
let result = match result {
Ok(value) => checked_hook_result(value),
Err(error) => error.into(),
};
Any::into_raw_ffi_any(result)
}
#[inline(always)]
fn with_def_region_kind<T>(
visitor: StructuralVisitorHandle,
kind: DefRegionKind,
callback: impl FnOnce() -> T,
) -> T {
unsafe {
let previous = *(*visitor).def_region_mode.get();
if previous == kind as i32 || previous == DefRegionKind::Pattern as i32 {
return callback();
}
*(*visitor).def_region_mode.get() = kind as i32;
struct Restore {
visitor: StructuralVisitorHandle,
previous: i32,
}
impl Drop for Restore {
#[inline(always)]
fn drop(&mut self) {
unsafe { *(*self.visitor).def_region_mode.get() = self.previous };
}
}
let _restore = Restore { visitor, previous };
callback()
}
}
}
#[inline(always)]
fn dispatcher_ptr<D>(dispatch: &mut D) -> *mut c_void {
std::ptr::from_mut(dispatch).cast()
}
#[inline(always)]
fn visit_result_raw(result: Result<Option<VisitInterrupt>>, raw: TVMFFIAny) -> TVMFFIAny {
match result {
Ok(None) => TVMFFIAny::new(),
Ok(Some(interrupt)) => Any::into_raw_ffi_any(interrupt.into()),
Err(error) => {
core::hint::cold_path();
StructuralVisitorObj::with_visit_context_raw(Any::into_raw_ffi_any(error.into()), raw)
}
}
}
#[inline(always)]
pub(super) fn visit_result_from_any(result: Any) -> Result<Option<VisitInterrupt>> {
let tag = result.type_index();
if tag == TVMFFITypeIndex::kTVMFFINone as i32 {
return Ok(None);
}
core::hint::cold_path();
if tag == TVMFFITypeIndex::kTVMFFIError as i32 {
Err(unsafe { take_error(result) })
} else {
use crate::type_traits::AnyCompatible;
let mut raw = Any::into_raw_ffi_any(result);
Ok(Some(unsafe {
VisitInterrupt::move_from_any_after_check(&mut raw)
}))
}
}
#[inline(always)]
fn checked_hook_result(value: Any) -> Any {
let tag = value.type_index();
if tag == TVMFFITypeIndex::kTVMFFINone as i32
|| tag == TVMFFITypeIndex::kTVMFFIError as i32
|| tag == TVMFFITypeIndex::kTVMFFIVisitInterrupt as i32
{
value
} else {
invalid_visit_hook_result_error().into()
}
}
#[inline(always)]
unsafe fn take_error(value: Any) -> Error {
use crate::type_traits::AnyCompatible;
let mut raw = Any::into_raw_ffi_any(value);
unsafe { Error::move_from_any_after_check(&mut raw) }
}
static STRUCTURAL_VISIT_COLUMN: ColdInitLazyLock<TypeAttrColumn> = ColdInitLazyLock::new(|| {
TypeAttrColumn::lookup(STRUCTURAL_VISIT_ATTR).expect("missing __s_visit__ column")
});
#[inline(always)]
fn structural_visit_column() -> TypeAttrColumn {
*STRUCTURAL_VISIT_COLUMN
}
#[cold]
#[inline(never)]
fn unregistered_visit_type_error(type_index: i32) -> Error {
runtime_error(&format!(
"native visitor: unregistered type index {type_index}"
))
}
#[cold]
#[inline(never)]
fn invalid_visit_hook_result_error() -> Error {
Error::new(
TYPE_ERROR,
"structural visit hook must return None or ffi.VisitInterrupt",
)
}