Skip to main content

tvm_ffi/extra/
structural_visit.rs

1/*
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *   http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing,
13 * software distributed under the License is distributed on an
14 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 * KIND, either express or implied.  See the License for the
16 * specific language governing permissions and limitations
17 * under the License.
18 */
19
20//! Native Rust structural visiting.
21//!
22//! Two public layers:
23//!
24//! * [`Dispatcher`](crate::Dispatcher) + [`structural_visit`] — handlers drive
25//!   recursion through an explicit [`StructuralVisitorView`] context. Pass state belongs on
26//!   the dispatcher; `visitor.visit(self, &child)` reborrows it for recursive calls.
27//!   Unmatched values use default child traversal.
28//! * [`structural_walk`](crate::structural_walk) — dispatcher or closure observers steer engine-owned
29//!   recursion through [`WalkResult`](crate::WalkResult) (advance, skip, interrupt).
30//!
31//! Both layers thread definition-region state explicitly. Walk handlers opt in
32//! with a trailing [`DefRegionKind`]; visit handlers read it from their context
33//! and may override it for a child through [`StructuralVisitorView::with_def_region_kind`].
34//!
35//! # Safety of the implementation
36//!
37//! The public interface is safe; its implementation deliberately uses raw pointers.
38//! Each private runtime has a matching dispatcher vtable and a fixed dispatcher
39//! identity. The dispatcher stays alive in the outer call, and the engine's dispatcher
40//! pointer publishes the actual exclusive reborrow before native recursion, then
41//! restores the enclosing borrow on return. Views and native runtime handles cannot
42//! escape their synchronous calls. No reference to the whole runtime spans recursion,
43//! and owning ABI results transfer exactly one reference at each boundary.
44//!
45//! Callbacks must return failures through [`Result`] and must not panic. A panic
46//! reaching the non-unwinding `extern "C"` callback boundary aborts the process;
47//! visitor callbacks do not catch or transport panic payloads across native frames.
48
49// `_raw` functions move one owned `TVMFFIAny`: forward it once, or import it into `Any` before
50// anything can unwind. Private `extern "C-unwind"` helpers return it in registers on AArch64.
51
52use std::cell::UnsafeCell;
53use std::marker::PhantomData;
54
55use super::dispatch::IntoDispatcher;
56use std::borrow::BorrowMut;
57use std::ops::ControlFlow;
58use std::os::raw::c_void;
59use std::rc::Rc;
60
61use super::cold_init_lazy_lock::ColdInitLazyLock;
62
63use crate::any::{Any, AnyView};
64use crate::derive::{Object, ObjectRef};
65use crate::error::{Error, Result, TYPE_ERROR};
66use crate::function::Function;
67use crate::object::{Object, ObjectArc};
68use crate::reflection::TypeAttrColumn;
69use crate::tvm_ffi_sys::{
70    TVMFFIAny, TVMFFIDefRegionKind, TVMFFIFieldInfo, TVMFFIGetTypeInfo, TVMFFIObject,
71    TVMFFITypeIndex,
72};
73
74use super::structural_common::{
75    FLAG_SEQ_HASH_IGNORE, field_def_region, for_each_field_info, free_var_child_region,
76    missing_field_getter_error, registered_type_index, runtime_error, seq_hash_kind,
77    wrong_dispatcher,
78};
79use super::visit_error_context::{enrich_visit_error, update_visit_error_context};
80
81const STRUCTURAL_VISIT_ATTR: &str = "__s_visit__";
82
83// -----------------------------------------------------------------------------
84// Data structures and FFI layout
85// -----------------------------------------------------------------------------
86
87type StructuralVisitorHandle = *mut StructuralVisitorObj;
88
89type FStructuralVisit =
90    unsafe extern "C" fn(StructuralVisitorHandle, AnyView<'static>) -> TVMFFIAny;
91
92/// The visitor vtable, laid out as the C++ one.
93#[repr(C)]
94struct StructuralVisitorVTable {
95    visit: FStructuralVisit,
96}
97
98/// The native visitor state, laid out as the prefix of the C++
99/// `StructuralVisitorObj`.
100#[repr(C)]
101pub struct StructuralVisitorObj {
102    base_: Object,
103    vtable: *const StructuralVisitorVTable,
104    def_region_mode: UnsafeCell<i32>,
105}
106
107// Scoped runtime prefixes are not independently allocatable ObjectCore values.
108impl StructuralVisitorObj {
109    const TYPE_KEY: &'static str = "ffi.StructuralVisitor";
110}
111
112impl Default for StructuralVisitorObj {
113    #[inline(always)]
114    fn default() -> Self {
115        Self {
116            base_: Object::new(),
117            vtable: std::ptr::null(),
118            def_region_mode: UnsafeCell::new(DefRegionKind::None as i32),
119        }
120    }
121}
122
123/// Native `ffi.VisitInterrupt` object containing the interruption payload.
124#[repr(C)]
125#[derive(Object)]
126#[object(
127    type_key = "ffi.VisitInterrupt",
128    unsafe type_index = TVMFFITypeIndex::kTVMFFIVisitInterrupt,
129    final
130)]
131pub struct VisitInterruptObj {
132    base_: Object,
133    /// Payload returned with the interrupt, or FFI `None` for no payload.
134    pub value: Any,
135}
136
137const _: () = {
138    assert!(
139        std::mem::offset_of!(StructuralVisitorObj, vtable) == std::mem::size_of::<TVMFFIObject>()
140    );
141    assert!(
142        std::mem::offset_of!(StructuralVisitorObj, def_region_mode)
143            == std::mem::size_of::<TVMFFIObject>() + std::mem::size_of::<*const c_void>()
144    );
145    assert!(std::mem::offset_of!(VisitInterruptObj, value) == std::mem::size_of::<TVMFFIObject>());
146};
147
148/// Definition-region state active at the current value.
149///
150/// Reflected fields marked `SEqHashDefPattern` or
151/// `SEqHashDefSimple` override the inherited state for that field's
152/// complete recursive visit.
153#[repr(i32)]
154#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
155pub enum DefRegionKind {
156    /// The value is outside a definition region.
157    #[default]
158    None = 0,
159    /// Definitions apply recursively through the visited value.
160    Pattern = 1,
161    /// Definitions apply to the visited value alone; its type is walked as uses.
162    Simple = 2,
163}
164
165const _: () = {
166    assert!(DefRegionKind::None as i32 == TVMFFIDefRegionKind::kTVMFFIDefRegionKindNone as i32);
167    assert!(
168        DefRegionKind::Pattern as i32 == TVMFFIDefRegionKind::kTVMFFIDefRegionKindPattern as i32
169    );
170    assert!(DefRegionKind::Simple as i32 == TVMFFIDefRegionKind::kTVMFFIDefRegionKindSimple as i32);
171};
172
173/// Interrupt state of a traversal, an `ffi.VisitInterrupt` object.
174///
175/// Entry points and visitor-layer calls return
176/// `Result<Option<VisitInterrupt>>`: `Ok(None)` means the (sub)graph was
177/// visited completely, `Ok(Some(..))` means a handler halted the traversal
178/// with this interrupt, and `Err` means it failed.
179///
180/// This is an owning object reference, like [`Error`]. Payload reads through
181/// `interrupt.value` borrow the shared [`VisitInterruptObj`]; use [`Self::new`]
182/// instead of a payload-only struct literal. Moving or replacing the payload
183/// through this reference is not supported; clone it to retain a separate owner.
184#[repr(C)]
185#[derive(Clone, ObjectRef)]
186pub struct VisitInterrupt {
187    data: ObjectArc<VisitInterruptObj>,
188}
189
190impl VisitInterrupt {
191    /// Create an interrupt carrying an FFI-compatible payload back to the caller of
192    /// the visit.
193    ///
194    /// # Arguments
195    ///
196    /// * `payload` - The payload, such as the value that was found.
197    ///
198    /// # Returns
199    ///
200    /// The interrupt.
201    #[inline(always)]
202    pub fn new<T: Into<Any>>(payload: T) -> Self {
203        Self::from_object(VisitInterruptObj {
204            base_: Object::new(),
205            value: payload.into(),
206        })
207    }
208}
209
210/// Native state and default traversal for [`StructuralVisitEngine`].
211/// The engine owns allocation, scoped dispatch and results.
212///
213/// # Safety
214///
215/// `Self` must have a complete initialized C layout with [`StructuralVisitorObj`]
216/// at offset zero, matching its registered native type and every field hooks access.
217/// Native-mutated fields require interior mutability; shared references must not
218/// freeze them during recursion. Hooks must preserve the vtable and valid region state.
219///
220/// The base must not implement [`crate::ObjectCore`] or otherwise allow callers
221/// to retain the runtime. Hooks may access the runtime and borrowed traversal
222/// state only synchronously, on the calling thread.
223///
224/// `default_visit_raw` returns one owning ABI None/Error/VisitInterrupt and recurses only
225/// through the installed vtable, without a whole-base mutable reference spanning
226/// recursion. Failures use that result carrier; panics must not cross native callbacks.
227pub unsafe trait StructuralVisitorBase: Sized {
228    /// Initialize the complete base before the engine installs its runtime header.
229    fn new() -> Result<Self>;
230
231    /// Native type registered for this complete base layout.
232    fn type_index() -> i32;
233
234    /// Descend from a borrowed value without dispatching that value again.
235    /// Return one owning ABI None, Error, or VisitInterrupt value.
236    /// Children reenter the installed vtable. The ordinary implementation accesses
237    /// only the visitor prefix, so specialized bases may reuse it.
238    ///
239    /// # Safety
240    /// `runtime` is this live initialized base with a matching installed vtable.
241    /// The input and runtime remain borrowed for this synchronous call. The
242    /// caller has published the current exclusive dispatcher reborrow.
243    unsafe fn default_visit_raw(runtime: *mut Self, value: AnyView<'_>) -> TVMFFIAny;
244}
245
246// SAFETY: this prefix exactly matches the ordinary native visitor. Its mutable
247// region is interior mutable, and it exposes no safe ownership conversion.
248unsafe impl StructuralVisitorBase for StructuralVisitorObj {
249    #[inline(always)]
250    fn new() -> Result<Self> {
251        Ok(Self::default())
252    }
253
254    fn type_index() -> i32 {
255        static TYPE_INDEX: ColdInitLazyLock<i32> =
256            ColdInitLazyLock::new(|| registered_type_index(StructuralVisitorObj::TYPE_KEY));
257        *TYPE_INDEX
258    }
259
260    #[inline(always)]
261    unsafe fn default_visit_raw(runtime: *mut Self, value: AnyView<'_>) -> TVMFFIAny {
262        unsafe { Self::visit_children_raw(runtime, *value.as_raw_ffi_any()) }
263    }
264}
265
266/// Scoped Rust visitor runtime with a statically selected native base.
267///
268/// Enter through [`StructuralVisitEngine::visit_root`]. The engine owns the complete base
269/// and the dispatcher reborrow handle, while the dispatcher stays in the outer
270/// call. Engines and their owning handles are never exposed to callbacks.
271///
272/// Select a custom base with `StructuralVisitEngine::<Base>::visit_root(root, input)`.
273/// Handlers use `StructuralVisitorView<'_, Target, Base>` for recursion and
274/// base-state inspection; normalization preserves the actual target's identity.
275#[repr(C)]
276pub struct StructuralVisitEngine<Base = StructuralVisitorObj> {
277    base: Base,
278    dispatcher: *mut c_void,
279}
280
281/// The running structural visit, passed to each handler.
282///
283/// A handler that matches a node decides which children are visited: call
284/// [`default_visit`](Self::default_visit) to visit every child,
285/// [`visit`](Self::visit) to visit one value, or neither to skip the children.
286/// Pass the handler's own `self` as `dispatch` to every method. A handler
287/// returns `Ok(None)` to continue, or `Ok(Some(interrupt))` to stop the whole
288/// visit with a [`VisitInterrupt`].
289///
290/// See also [`structural_visit`].
291pub struct StructuralVisitorView<'a, D, Base = StructuralVisitorObj> {
292    runtime: StructuralVisitorHandle,
293    #[allow(clippy::type_complexity)]
294    _marker: PhantomData<(&'a mut (), fn(&mut D) -> Base, Rc<()>)>,
295}
296
297// -----------------------------------------------------------------------------
298// Public visitor API and entrypoint
299// -----------------------------------------------------------------------------
300
301impl<D, Base: StructuralVisitorBase> StructuralVisitorView<'_, D, Base> {
302    /// Inspect the base state; the borrow cannot outlive this view.
303    #[inline(always)]
304    pub fn base(&self) -> &Base {
305        // SAFETY: the unsafe base contract permits shared inspection, including
306        // native mutation through interior-mutable fields during recursion.
307        unsafe { &*self.runtime.cast::<Base>() }
308    }
309
310    /// Return the definition region the current value is in.
311    ///
312    /// Reflected fields marked as definitions set it while their values are
313    /// visited; it is [`DefRegionKind::None`] elsewhere.
314    ///
315    /// # Returns
316    ///
317    /// The current definition region.
318    #[inline(always)]
319    pub fn def_region_kind(&self) -> DefRegionKind {
320        // SAFETY: the view's runtime is live.
321        unsafe { std::mem::transmute(*(*self.runtime).def_region_mode.get()) }
322    }
323
324    /// Run `callback` under a scoped definition region.
325    ///
326    /// An enclosing [`DefRegionKind::Pattern`] region keeps precedence; the
327    /// previous region is restored on all exits.
328    ///
329    /// # Arguments
330    ///
331    /// * `kind` - The definition region to enter.
332    /// * `callback` - Runs inside the region, with this view.
333    ///
334    /// # Returns
335    ///
336    /// The result of `callback`.
337    #[inline(always)]
338    pub fn with_def_region_kind<R>(
339        &mut self,
340        kind: DefRegionKind,
341        callback: impl FnOnce(&mut Self) -> R,
342    ) -> R {
343        // SAFETY: the view's runtime outlives the callback.
344        StructuralVisitorObj::with_def_region_kind(self.runtime, kind, || callback(self))
345    }
346
347    /// Visit `value` through the dispatcher, as if the traversal had reached it.
348    ///
349    /// # Arguments
350    ///
351    /// * `dispatch` - The handler's own `self`.
352    /// * `value` - The value to visit.
353    ///
354    /// # Returns
355    ///
356    /// `None` to continue, or the interrupt that stopped the visit.
357    ///
358    /// # Errors
359    ///
360    /// Returns the error of the handler that `value` reaches.
361    #[inline(always)]
362    pub fn visit<'input, T>(&mut self, dispatch: &mut D, value: T) -> Result<Option<VisitInterrupt>>
363    where
364        T: Into<AnyView<'input>>,
365    {
366        let value: AnyView<'input> = value.into();
367        // SAFETY: the view ties this runtime to D; the check establishes the
368        // instance, and publishing uses the caller's actual exclusive reborrow.
369        unsafe {
370            visit_result_from_any(Any::from_raw_ffi_any(
371                StructuralVisitEngine::<Base>::visit_raw(
372                    self.runtime,
373                    dispatch,
374                    *value.as_raw_ffi_any(),
375                ),
376            ))
377        }
378    }
379
380    /// Visit the children of `value` through the dispatcher, without dispatching
381    /// `value` itself again.
382    ///
383    /// Children are found by the type's registered visit hook or its reflected
384    /// fields. A hook uses the visitor only through its arguments, during the
385    /// call, and must not keep it or call it outside them. The visit stops at the first child that returns an interrupt.
386    ///
387    /// # Arguments
388    ///
389    /// * `dispatch` - The handler's own `self`.
390    /// * `value` - The value whose children to visit.
391    ///
392    /// # Returns
393    ///
394    /// `None` to continue, or the interrupt that stopped the visit.
395    ///
396    /// # Errors
397    ///
398    /// Returns the first error of a child's handler.
399    #[inline(always)]
400    pub fn default_visit<'input, T>(
401        &self,
402        dispatch: &mut D,
403        value: T,
404    ) -> Result<Option<VisitInterrupt>>
405    where
406        T: Into<AnyView<'input>>,
407    {
408        // SAFETY: type/layout and scope come from the private view constructor;
409        // the supplied borrow must identify this runtime's dispatcher instance.
410        unsafe {
411            let context = StructuralVisitEngine::<Base>::checked_dispatcher(self.runtime, dispatch);
412            let value: AnyView<'input> = value.into();
413            visit_result_from_any(Any::from_raw_ffi_any(
414                StructuralVisitEngine::<Base>::default_visit_raw(
415                    self.runtime,
416                    context,
417                    *value.as_raw_ffi_any(),
418                ),
419            ))
420        }
421    }
422}
423
424/// Visit `root` with an owned or borrowed [`Dispatcher`](crate::Dispatcher).
425///
426/// Handlers receive [`StructuralVisitorView`] for explicit recursion and keep state on the
427/// dispatcher. Unmatched values use default child traversal. Pass `&mut D` to
428/// keep the dispatcher and inspect its state after the visit. An owned dispatcher
429/// is held in the outer call and uses the same borrowed traversal engine.
430///
431/// Callbacks must report failures through [`Result`] and must not panic: a panic
432/// reaching the non-unwinding native callback boundary aborts the process.
433///
434/// A callback must not modify an `ffi.List` or `ffi.Dict` that holds the
435/// value it receives or one of that value's ancestors: the traversal borrows
436/// their cells while the callback runs.
437///
438/// # Arguments
439///
440/// * `root` - Value borrowed for traversal, convertible into an [`AnyView`].
441/// * `dispatcher` - Owned or mutably borrowed dispatcher implementing [`IntoDispatcher`].
442///   Matching handlers control recursion through [`StructuralVisitorView`];
443///   unmatched values use default child traversal.
444///
445/// # Examples
446///
447/// Sum integer leaves while default traversal descends into the array:
448///
449/// ```
450/// # fn main() -> tvm_ffi::Result<()> {
451/// use tvm_ffi::{dispatch, structural_visit, AnyView, Array, Result, VisitInterrupt};
452/// use tvm_ffi::StructuralVisitorView;
453///
454/// struct Sum(i64);
455/// #[dispatch(visit, (AnyView<'_>, &mut StructuralVisitorView<'_, Self>), Result<Option<VisitInterrupt>>)]
456/// impl Sum {
457///     fn visit_integer(
458///         &mut self,
459///         value: i64,
460///         _: &mut StructuralVisitorView<'_, Self>,
461///     ) -> Result<Option<VisitInterrupt>> {
462///         self.0 += value;
463///         Ok(None)
464///     }
465/// }
466///
467/// let root = Array::from_vec(vec![1_i64, 2, 3]);
468/// let mut sum = Sum(0);
469/// assert!(structural_visit(&root, &mut sum)?.is_none());
470/// assert_eq!(sum.0, 6);
471/// # Ok(())
472/// # }
473/// ```
474pub fn structural_visit<'input, R, H, M, D, O>(
475    root: R,
476    dispatcher: H,
477) -> Result<Option<VisitInterrupt>>
478where
479    R: Into<AnyView<'input>>,
480    O: BorrowMut<D>,
481    for<'v, 'c, 'r> H: IntoDispatcher<
482            (
483                AnyView<'v>,
484                &'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
485            ),
486            Option<Result<Option<VisitInterrupt>>>,
487            M,
488            Target = D,
489            Output = O,
490        >,
491    for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
492            (
493                AnyView<'v>,
494                &'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
495            ),
496            Option<Result<Option<VisitInterrupt>>>,
497        >,
498{
499    StructuralVisitEngine::visit_root(root, dispatcher)
500}
501
502impl super::dispatch::CallbackResult<Result<Option<VisitInterrupt>>>
503    for Result<Option<VisitInterrupt>>
504{
505    #[inline(always)]
506    fn callback_result(self) -> Result<Option<VisitInterrupt>> {
507        self
508    }
509}
510
511// -----------------------------------------------------------------------------
512// StructuralVisitEngine implementation
513// -----------------------------------------------------------------------------
514
515// Generic runtime: only this implementation projects the dispatcher after Base.
516impl<Base: StructuralVisitorBase> StructuralVisitEngine<Base> {
517    /// Visit `root` with `dispatcher` on the native base `Base`. See also [`structural_visit`].
518    #[inline(always)]
519    pub fn visit_root<'input, R, H, M, D, O>(
520        root: R,
521        dispatcher: H,
522    ) -> Result<Option<VisitInterrupt>>
523    where
524        R: Into<AnyView<'input>>,
525        O: BorrowMut<D>,
526        for<'v, 'c, 'r> H: IntoDispatcher<
527                (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
528                Option<Result<Option<VisitInterrupt>>>,
529                M,
530                Target = D,
531                Output = O,
532            >,
533        for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
534                (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
535                Option<Result<Option<VisitInterrupt>>>,
536            >,
537    {
538        // Normalize the input once. Named dispatchers keep their own recursive view
539        // type; callback and tuple targets are their aggregate adapters.
540        let mut storage = dispatcher.into_dispatcher();
541        let root = root.into();
542        Self::run_visitor::<D>(*root.as_raw_ffi_any(), storage.borrow_mut())
543    }
544
545    /// The engine whose first field is the native prefix `runtime`.
546    ///
547    /// # Safety
548    /// `runtime` must belong to the live, synchronous Rust traversal that created
549    /// this exact repr(C) engine. An arbitrary native visitor is not such an engine.
550    #[inline(always)]
551    unsafe fn from_handle(runtime: StructuralVisitorHandle) -> *mut Self {
552        runtime.cast()
553    }
554
555    /// Publish the current exclusive `dispatch` reborrow and visit borrowed `raw`.
556    /// Return one owning raw None/interrupt/Error token.
557    ///
558    /// # Safety
559    /// `runtime` must belong to this live `D` invocation and `raw` remain valid
560    /// through recursion; the caller cannot use competing dispatcher borrows.
561    #[inline(always)]
562    unsafe fn visit_raw<D>(
563        runtime: StructuralVisitorHandle,
564        dispatch: &mut D,
565        raw: TVMFFIAny,
566    ) -> TVMFFIAny {
567        let context = unsafe { Self::checked_dispatcher(runtime, dispatch) };
568        unsafe { (*Self::from_handle(runtime)).dispatcher = context };
569        StructuralVisitorObj::call_visitor_raw(runtime, raw)
570    }
571
572    /// Check runtime identity and return the current `dispatch` reborrow pointer.
573    ///
574    /// # Safety
575    /// `runtime` must be the live engine for `D`; address equality does not establish
576    /// type or lifetime validity. Mismatch panics before a pointer is published.
577    // The caller has already established matching runtime/D types and scope.
578    // Read only the stored raw address: do not create a competing mutable borrow.
579    #[inline(always)]
580    unsafe fn checked_dispatcher<D>(
581        runtime: StructuralVisitorHandle,
582        dispatch: &mut D,
583    ) -> *mut c_void {
584        let current = dispatcher_ptr(dispatch);
585        if current != unsafe { (*Self::from_handle(runtime)).dispatcher } {
586            core::hint::cold_path();
587            wrong_dispatcher();
588        }
589        current
590    }
591
592    /// Build the native visitor callback for concrete dispatcher `D`. Its higher-ranked
593    /// view arguments restrict all borrows to the synchronous handler invocation.
594    #[inline(always)]
595    fn vtable<D>() -> &'static StructuralVisitorVTable
596    where
597        for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
598                (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
599                Option<Result<Option<VisitInterrupt>>>,
600            >,
601    {
602        &StructuralVisitorVTable {
603            visit: Self::rust_vtable_visit_raw::<D>,
604        }
605    }
606
607    /// Enter a native visitor callback with borrowed `value`; return one owning token.
608    /// Restore the incoming dispatcher provenance before returning to the caller.
609    ///
610    /// # Safety
611    /// `runtime` must have this `D` vtable and a live published reborrow. The erased
612    /// ABI lifetime on `value` is valid only for this call. Callback failures use
613    /// Result; Rust panics cannot cross this native C boundary.
614    unsafe extern "C" fn rust_vtable_visit_raw<D>(
615        runtime: StructuralVisitorHandle,
616        value: AnyView<'static>,
617    ) -> TVMFFIAny
618    where
619        for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
620                (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
621                Option<Result<Option<VisitInterrupt>>>,
622            >,
623    {
624        let context = unsafe { (*Self::from_handle(runtime)).dispatcher };
625        let raw = *value.as_raw_ffi_any();
626        let result = if raw.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
627            core::hint::cold_path();
628            TVMFFIAny::new()
629        } else {
630            let dispatch = unsafe { &mut *context.cast::<D>() };
631            let mut visitor = unsafe { Self::visitor_context::<D>(runtime) };
632            // SAFETY: the native caller keeps the input alive for this synchronous
633            // dispatch, and the handler does not modify a list or dict cell
634            // holding it (the callback contract); the higher-ranked callback
635            // cannot retain its borrowed view.
636            match unsafe {
637                crate::Dispatcher::dispatch_tuple(
638                    &mut *dispatch,
639                    (AnyView::from_raw_ffi_any(raw), &mut visitor),
640                )
641            } {
642                Some(result) => visit_result_raw(result, raw),
643                None => unsafe { Self::default_visit_raw(runtime, dispatcher_ptr(dispatch), raw) },
644            }
645        };
646        // The enclosing callback may resume using its ancestor mutable borrow.
647        // Native hooks cannot restore Rust-only borrow provenance for us.
648        unsafe { (*Self::from_handle(runtime)).dispatcher = context };
649        result
650    }
651
652    /// Create the handler view for borrowed `runtime`, without retaining it.
653    ///
654    /// # Safety
655    /// The live runtime's vtable must match D. Keep this view within the current
656    /// higher-ranked handler call, never outliving the runtime or dispatcher.
657    #[inline(always)]
658    unsafe fn visitor_context<'a, D>(
659        runtime: StructuralVisitorHandle,
660    ) -> StructuralVisitorView<'a, D, Base> {
661        StructuralVisitorView {
662            runtime,
663            _marker: PhantomData,
664        }
665    }
666
667    /// Publish `context` and descend through borrowed `raw`, then annotate errors.
668    /// The raw return transfers one owning None/interrupt/Error token.
669    ///
670    /// # Safety
671    /// `runtime` must be live and `context` its current exclusive dispatcher reborrow;
672    /// `raw` remains borrowed and must survive the complete synchronous descent.
673    #[inline(always)]
674    unsafe fn default_visit_raw(
675        runtime: StructuralVisitorHandle,
676        context: *mut c_void,
677        raw: TVMFFIAny,
678    ) -> TVMFFIAny {
679        // Context is the supplied exclusive reborrow. Publish it before descent;
680        // the handle-only scanner never creates or uses a competing D reference.
681        unsafe { (*Self::from_handle(runtime)).dispatcher = context };
682        unsafe {
683            StructuralVisitorObj::with_visit_context_raw(
684                Base::default_visit_raw(runtime.cast(), AnyView::from_raw_ffi_any(raw)),
685                raw,
686            )
687        }
688    }
689
690    /// Visit borrowed `root` with `dispatch` on a fresh engine; the root's owner outlives it.
691    pub(super) fn run_visitor<D>(
692        root: TVMFFIAny,
693        dispatch: &mut D,
694    ) -> Result<Option<VisitInterrupt>>
695    where
696        for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
697                (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
698                Option<Result<Option<VisitInterrupt>>>,
699            >,
700    {
701        let base = Base::new()?;
702        let type_index = Base::type_index();
703        // The managed runtime may borrow its dispatcher only for this call.
704        // Trusted hooks release all temporary handles before the owner is dropped.
705        let mut engine = unsafe {
706            ObjectArc::<Object>::new_opaque_unchecked(
707                StructuralVisitEngine {
708                    base,
709                    dispatcher: dispatcher_ptr(dispatch),
710                },
711                type_index,
712            )
713        };
714        let runtime = ObjectArc::as_mut_ptr(&mut engine).cast::<StructuralVisitorObj>();
715        // The engine and outer dispatcher storage stay stationary for the whole run.
716        // Only raw field projections cross recursion; no whole-engine borrow survives.
717        unsafe {
718            (*runtime).vtable = Self::vtable::<D>();
719            *(*runtime).def_region_mode.get() = DefRegionKind::None as i32;
720        }
721        // Keep the result owned while releasing the runtime, including on unwind.
722        let result =
723            unsafe { Any::from_raw_ffi_any(StructuralVisitorObj::call_visitor_raw(runtime, root)) };
724        // Trusted hooks have released every temporary strong/weak runtime reference.
725        drop(engine);
726        // The public traversal boundary, as the C++ `StructuralVisitExpected`.
727        visit_result_from_any(result).map_err(|error| enrich_visit_error(error, root))
728    }
729}
730
731// -----------------------------------------------------------------------------
732// Prefix-only default traversal and native ABI operations
733// -----------------------------------------------------------------------------
734
735impl StructuralVisitorObj {
736    /// Consume `result`, naming the borrowed `value` in it when it is an Error.
737    #[inline(always)]
738    fn with_visit_context_raw(result: TVMFFIAny, value: TVMFFIAny) -> TVMFFIAny {
739        if result.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
740            result
741        } else {
742            Self::with_visit_stop_context_raw(result, value)
743        }
744    }
745
746    // Continuing visits need no owning conversion or input inspection. Keep
747    // interrupt/error handling out of their recursive frames.
748    #[cold]
749    #[inline(never)]
750    extern "C-unwind" fn with_visit_stop_context_raw(
751        result: TVMFFIAny,
752        value: TVMFFIAny,
753    ) -> TVMFFIAny {
754        if result.type_index != TVMFFITypeIndex::kTVMFFIError as i32
755            || value.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32
756        {
757            result
758        } else {
759            // SAFETY: the Error tag establishes the owned object's representation.
760            unsafe {
761                let error = take_error(Any::from_raw_ffi_any(result));
762                Any::into_raw_ffi_any(update_visit_error_context(error, value).into())
763            }
764        }
765    }
766
767    /// Visit the children of `value` through its hook or reflected fields, without
768    /// dispatching `value` itself.
769    ///
770    /// # Safety
771    ///
772    /// `runtime` is live with its dispatcher published, and `value` outlives descent.
773    #[inline(always)]
774    unsafe fn visit_children_raw(runtime: StructuralVisitorHandle, value: TVMFFIAny) -> TVMFFIAny {
775        if let Some(attr) = structural_visit_column().get_raw_ffi_any(value.type_index)
776            && attr.type_index != TVMFFITypeIndex::kTVMFFINone as i32
777        {
778            return unsafe { Self::call_structural_visit_hook_raw(runtime, value, attr) };
779        }
780
781        if value.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
782            TVMFFIAny::new()
783        } else {
784            unsafe { Self::visit_reflected_fields_raw(runtime, value) }
785        }
786    }
787
788    /// Visit each reflected field of the object in `value`.
789    ///
790    /// # Safety
791    ///
792    /// As [`Self::visit_children_raw`]. Outlined (frequent, not cold) to keep reflection
793    /// scratch out of recursive frames.
794    #[inline(never)]
795    unsafe extern "C-unwind" fn visit_reflected_fields_raw(
796        runtime: StructuralVisitorHandle,
797        value: TVMFFIAny,
798    ) -> TVMFFIAny {
799        let type_info = unsafe { TVMFFIGetTypeInfo(value.type_index) };
800        if type_info.is_null() {
801            return Any::into_raw_ffi_any(unregistered_visit_type_error(value.type_index).into());
802        }
803        let def_region_kind = unsafe {
804            free_var_child_region(
805                std::mem::transmute::<i32, DefRegionKind>(*(*runtime).def_region_mode.get()),
806                seq_hash_kind(type_info),
807            )
808        };
809        let object = unsafe { value.data_union.v_obj } as *mut u8;
810        let halted = unsafe {
811            for_each_field_info(type_info, &mut |field| {
812                let result = Self::visit_reflected_field(runtime, object, field, def_region_kind);
813                if result.is_none() {
814                    ControlFlow::Continue(())
815                } else {
816                    ControlFlow::Break(result)
817                }
818            })
819        };
820        Any::into_raw_ffi_any(halted.unwrap_or_else(Any::new))
821    }
822
823    /// Visit one reflected `field` of `object` under `inherited_region`.
824    ///
825    /// # Safety
826    ///
827    /// `runtime` and `object` are live, and `field` describes `object`'s storage.
828    unsafe fn visit_reflected_field(
829        runtime: StructuralVisitorHandle,
830        object: *mut u8,
831        field: &TVMFFIFieldInfo,
832        inherited_region: DefRegionKind,
833    ) -> Any {
834        if field.flags & FLAG_SEQ_HASH_IGNORE != 0 {
835            return Any::new();
836        }
837
838        let Some(getter) = field.getter else {
839            return Any::from(unsafe {
840                missing_field_getter_error("visitor", field.name.as_str())
841            });
842        };
843        let address = unsafe { object.offset(field.offset as isize) as *mut c_void };
844        // Own the getter result so partial writes and recursive borrows drop safely.
845        let mut child = Any::new();
846        if unsafe { getter(address, Any::as_raw_ffi_any_mut(&mut child)) } != 0 {
847            return Any::from(Error::from_raised());
848        }
849
850        let borrowed = AnyView::from(&child);
851        let child_region = field_def_region(field, inherited_region);
852        Self::with_def_region_kind(runtime, child_region, || unsafe {
853            Any::from_raw_ffi_any(Self::call_visitor_raw(runtime, *borrowed.as_raw_ffi_any()))
854        })
855    }
856
857    /// Dispatch borrowed `value` through the runtime's vtable.
858    #[inline(always)]
859    fn call_visitor_raw(runtime: StructuralVisitorHandle, value: TVMFFIAny) -> TVMFFIAny {
860        if value.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
861            core::hint::cold_path();
862            return TVMFFIAny::new();
863        }
864        unsafe {
865            let callback = (*(*runtime).vtable).visit;
866            callback(runtime, AnyView::from_raw_ffi_any(value))
867        }
868    }
869
870    /// Call the registered visit `hook` (native pointer or packed function) on `value`.
871    ///
872    /// # Safety
873    ///
874    /// `runtime` is live with its dispatcher published, `value` and `hook` outlive the
875    /// call, and a native hook has the visit ABI for this runtime's layout and obeys
876    /// [`StructuralVisitorBase`]'s non-escaping contract.
877    #[inline(always)]
878    pub unsafe fn call_structural_visit_hook_raw(
879        runtime: *mut Self,
880        value: TVMFFIAny,
881        hook: TVMFFIAny,
882    ) -> TVMFFIAny {
883        unsafe {
884            if hook.type_index == TVMFFITypeIndex::kTVMFFIOpaquePtr as i32 {
885                let pointer = hook.data_union.v_ptr;
886                // Native pointer registration is an unsafe contract: a non-null
887                // callback with this ABI, borrowed runtime lifetime and owning result.
888                let hook: FStructuralVisit = std::mem::transmute(pointer);
889                hook(runtime, AnyView::from_raw_ffi_any(value))
890            } else {
891                core::hint::cold_path();
892                Self::call_structural_visit_hook_slow_raw(runtime, value, hook)
893            }
894        }
895    }
896
897    /// The packed-function arm of [`Self::call_structural_visit_hook_raw`], outlined as cold;
898    /// its result tag is checked.
899    #[cold]
900    #[inline(never)]
901    extern "C-unwind" fn call_structural_visit_hook_slow_raw(
902        runtime: StructuralVisitorHandle,
903        value: TVMFFIAny,
904        hook: TVMFFIAny,
905    ) -> TVMFFIAny {
906        // Packed hooks (including Python class methods) can return any well-formed
907        // Any through safe APIs. Their result tag needs validation before decoding.
908        let result = unsafe {
909            Function::try_from(AnyView::from_raw_ffi_any(hook)).and_then(|function| {
910                // Borrow the live runtime for this synchronous hook; it may not retain it.
911                let object = runtime.cast::<TVMFFIObject>();
912                let mut runtime = TVMFFIAny::new();
913                runtime.type_index = (*object).type_index;
914                runtime.data_union.v_obj = object;
915                function.call_packed(&[
916                    AnyView::from_raw_ffi_any(runtime),
917                    AnyView::from_raw_ffi_any(value),
918                ])
919            })
920        };
921        let result = match result {
922            Ok(value) => checked_hook_result(value),
923            Err(error) => error.into(),
924        };
925        Any::into_raw_ffi_any(result)
926    }
927
928    /// Run `callback` under requested `kind` on live borrowed `visitor`, preserving
929    /// Pattern precedence. Restore prior region on return or Rust unwind. Internal
930    /// callers keep the runtime live and retain no reference to its region field.
931    #[inline(always)]
932    fn with_def_region_kind<T>(
933        visitor: StructuralVisitorHandle,
934        kind: DefRegionKind,
935        callback: impl FnOnce() -> T,
936    ) -> T {
937        unsafe {
938            let previous = *(*visitor).def_region_mode.get();
939            if previous == kind as i32 || previous == DefRegionKind::Pattern as i32 {
940                // Inherited descent does not change state; Pattern keeps precedence.
941                return callback();
942            }
943            *(*visitor).def_region_mode.get() = kind as i32;
944            struct Restore {
945                visitor: StructuralVisitorHandle,
946                previous: i32,
947            }
948            impl Drop for Restore {
949                #[inline(always)]
950                fn drop(&mut self) {
951                    unsafe { *(*self.visitor).def_region_mode.get() = self.previous };
952                }
953            }
954            let _restore = Restore { visitor, previous };
955            callback()
956        }
957    }
958}
959
960// -----------------------------------------------------------------------------
961// Private utility functions
962// -----------------------------------------------------------------------------
963
964#[inline(always)]
965fn dispatcher_ptr<D>(dispatch: &mut D) -> *mut c_void {
966    std::ptr::from_mut(dispatch).cast()
967}
968
969// The visit result carrier, as C++: None continues; a VisitInterrupt or an Error stops.
970#[inline(always)]
971fn visit_result_raw(result: Result<Option<VisitInterrupt>>, raw: TVMFFIAny) -> TVMFFIAny {
972    match result {
973        Ok(None) => TVMFFIAny::new(),
974        Ok(Some(interrupt)) => Any::into_raw_ffi_any(interrupt.into()),
975        Err(error) => {
976            core::hint::cold_path();
977            StructuralVisitorObj::with_visit_context_raw(Any::into_raw_ffi_any(error.into()), raw)
978        }
979    }
980}
981
982#[inline(always)]
983pub(super) fn visit_result_from_any(result: Any) -> Result<Option<VisitInterrupt>> {
984    let tag = result.type_index();
985    if tag == TVMFFITypeIndex::kTVMFFINone as i32 {
986        return Ok(None);
987    }
988    core::hint::cold_path();
989    if tag == TVMFFITypeIndex::kTVMFFIError as i32 {
990        // SAFETY: internal results obey the documented owning carrier protocol.
991        Err(unsafe { take_error(result) })
992    } else {
993        // SAFETY: the owning result protocol established a non-null interrupt.
994        // Transfer that same reference, including carriers shared by native hooks.
995        use crate::type_traits::AnyCompatible;
996        let mut raw = Any::into_raw_ffi_any(result);
997        Ok(Some(unsafe {
998            VisitInterrupt::move_from_any_after_check(&mut raw)
999        }))
1000    }
1001}
1002
1003// A packed hook's owning Any is valid, but its result type is dynamic.
1004#[inline(always)]
1005fn checked_hook_result(value: Any) -> Any {
1006    let tag = value.type_index();
1007    if tag == TVMFFITypeIndex::kTVMFFINone as i32
1008        || tag == TVMFFITypeIndex::kTVMFFIError as i32
1009        || tag == TVMFFITypeIndex::kTVMFFIVisitInterrupt as i32
1010    {
1011        value
1012    } else {
1013        invalid_visit_hook_result_error().into()
1014    }
1015}
1016
1017/// Consume exactly one Error reference after its integer tag was checked.
1018#[inline(always)]
1019unsafe fn take_error(value: Any) -> Error {
1020    use crate::type_traits::AnyCompatible;
1021    let mut raw = Any::into_raw_ffi_any(value);
1022    unsafe { Error::move_from_any_after_check(&mut raw) }
1023}
1024
1025// Columns are process-lifetime registry entries; a missing per-type cell still
1026// falls back to reflected traversal. A missing required column is an error.
1027static STRUCTURAL_VISIT_COLUMN: ColdInitLazyLock<TypeAttrColumn> = ColdInitLazyLock::new(|| {
1028    TypeAttrColumn::lookup(STRUCTURAL_VISIT_ATTR).expect("missing __s_visit__ column")
1029});
1030
1031/// Return the shared visit-hook column without caching optional per-type cells.
1032#[inline(always)]
1033fn structural_visit_column() -> TypeAttrColumn {
1034    *STRUCTURAL_VISIT_COLUMN
1035}
1036
1037// -----------------------------------------------------------------------------
1038// Outlined errors
1039// -----------------------------------------------------------------------------
1040
1041#[cold]
1042#[inline(never)]
1043fn unregistered_visit_type_error(type_index: i32) -> Error {
1044    runtime_error(&format!(
1045        "native visitor: unregistered type index {type_index}"
1046    ))
1047}
1048
1049#[cold]
1050#[inline(never)]
1051fn invalid_visit_hook_result_error() -> Error {
1052    Error::new(
1053        TYPE_ERROR,
1054        "structural visit hook must return None or ffi.VisitInterrupt",
1055    )
1056}