tvm_ffi/extra/structural_visit.rs
1/*
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing,
13 * software distributed under the License is distributed on an
14 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 * KIND, either express or implied. See the License for the
16 * specific language governing permissions and limitations
17 * under the License.
18 */
19
20//! Native Rust structural visiting.
21//!
22//! Two public layers:
23//!
24//! * [`Dispatcher`](crate::Dispatcher) + [`structural_visit`] — handlers drive
25//! recursion through an explicit [`StructuralVisitorView`] context. Pass state belongs on
26//! the dispatcher; `visitor.visit(self, &child)` reborrows it for recursive calls.
27//! Unmatched values use default child traversal.
28//! * [`structural_walk`](crate::structural_walk) — dispatcher or closure observers steer engine-owned
29//! recursion through [`WalkResult`](crate::WalkResult) (advance, skip, interrupt).
30//!
31//! Both layers thread definition-region state explicitly. Walk handlers opt in
32//! with a trailing [`DefRegionKind`]; visit handlers read it from their context
33//! and may override it for a child through [`StructuralVisitorView::with_def_region_kind`].
34//!
35//! # Safety of the implementation
36//!
37//! The public interface is safe; its implementation deliberately uses raw pointers.
38//! Each private runtime has a matching dispatcher vtable and a fixed dispatcher
39//! identity. The dispatcher stays alive in the outer call, and the engine's dispatcher
40//! pointer publishes the actual exclusive reborrow before native recursion, then
41//! restores the enclosing borrow on return. Views and native runtime handles cannot
42//! escape their synchronous calls. No reference to the whole runtime spans recursion,
43//! and owning ABI results transfer exactly one reference at each boundary.
44//!
45//! Callbacks must return failures through [`Result`] and must not panic. A panic
46//! reaching the non-unwinding `extern "C"` callback boundary aborts the process;
47//! visitor callbacks do not catch or transport panic payloads across native frames.
48
49// `_raw` functions move one owned `TVMFFIAny`: forward it once, or import it into `Any` before
50// anything can unwind. Private `extern "C-unwind"` helpers return it in registers on AArch64.
51
52use std::cell::UnsafeCell;
53use std::marker::PhantomData;
54
55use super::dispatch::IntoDispatcher;
56use std::borrow::BorrowMut;
57use std::ops::ControlFlow;
58use std::os::raw::c_void;
59use std::rc::Rc;
60
61use super::cold_init_lazy_lock::ColdInitLazyLock;
62
63use crate::any::{Any, AnyView};
64use crate::derive::{Object, ObjectRef};
65use crate::error::{Error, Result, TYPE_ERROR};
66use crate::function::Function;
67use crate::object::{Object, ObjectArc};
68use crate::reflection::TypeAttrColumn;
69use crate::tvm_ffi_sys::{
70 TVMFFIAny, TVMFFIDefRegionKind, TVMFFIFieldInfo, TVMFFIGetTypeInfo, TVMFFIObject,
71 TVMFFITypeIndex,
72};
73
74use super::structural_common::{
75 FLAG_SEQ_HASH_IGNORE, field_def_region, for_each_field_info, free_var_child_region,
76 missing_field_getter_error, registered_type_index, runtime_error, seq_hash_kind,
77 wrong_dispatcher,
78};
79use super::visit_error_context::{enrich_visit_error, update_visit_error_context};
80
81const STRUCTURAL_VISIT_ATTR: &str = "__s_visit__";
82
83// -----------------------------------------------------------------------------
84// Data structures and FFI layout
85// -----------------------------------------------------------------------------
86
87type StructuralVisitorHandle = *mut StructuralVisitorObj;
88
89type FStructuralVisit =
90 unsafe extern "C" fn(StructuralVisitorHandle, AnyView<'static>) -> TVMFFIAny;
91
92/// The visitor vtable, laid out as the C++ one.
93#[repr(C)]
94struct StructuralVisitorVTable {
95 visit: FStructuralVisit,
96}
97
98/// The native visitor state, laid out as the prefix of the C++
99/// `StructuralVisitorObj`.
100#[repr(C)]
101pub struct StructuralVisitorObj {
102 base_: Object,
103 vtable: *const StructuralVisitorVTable,
104 def_region_mode: UnsafeCell<i32>,
105}
106
107// Scoped runtime prefixes are not independently allocatable ObjectCore values.
108impl StructuralVisitorObj {
109 const TYPE_KEY: &'static str = "ffi.StructuralVisitor";
110}
111
112impl Default for StructuralVisitorObj {
113 #[inline(always)]
114 fn default() -> Self {
115 Self {
116 base_: Object::new(),
117 vtable: std::ptr::null(),
118 def_region_mode: UnsafeCell::new(DefRegionKind::None as i32),
119 }
120 }
121}
122
123/// Native `ffi.VisitInterrupt` object containing the interruption payload.
124#[repr(C)]
125#[derive(Object)]
126#[object(
127 type_key = "ffi.VisitInterrupt",
128 unsafe type_index = TVMFFITypeIndex::kTVMFFIVisitInterrupt,
129 final
130)]
131pub struct VisitInterruptObj {
132 base_: Object,
133 /// Payload returned with the interrupt, or FFI `None` for no payload.
134 pub value: Any,
135}
136
137const _: () = {
138 assert!(
139 std::mem::offset_of!(StructuralVisitorObj, vtable) == std::mem::size_of::<TVMFFIObject>()
140 );
141 assert!(
142 std::mem::offset_of!(StructuralVisitorObj, def_region_mode)
143 == std::mem::size_of::<TVMFFIObject>() + std::mem::size_of::<*const c_void>()
144 );
145 assert!(std::mem::offset_of!(VisitInterruptObj, value) == std::mem::size_of::<TVMFFIObject>());
146};
147
148/// Definition-region state active at the current value.
149///
150/// Reflected fields marked `SEqHashDefPattern` or
151/// `SEqHashDefSimple` override the inherited state for that field's
152/// complete recursive visit.
153#[repr(i32)]
154#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
155pub enum DefRegionKind {
156 /// The value is outside a definition region.
157 #[default]
158 None = 0,
159 /// Definitions apply recursively through the visited value.
160 Pattern = 1,
161 /// Definitions apply to the visited value alone; its type is walked as uses.
162 Simple = 2,
163}
164
165const _: () = {
166 assert!(DefRegionKind::None as i32 == TVMFFIDefRegionKind::kTVMFFIDefRegionKindNone as i32);
167 assert!(
168 DefRegionKind::Pattern as i32 == TVMFFIDefRegionKind::kTVMFFIDefRegionKindPattern as i32
169 );
170 assert!(DefRegionKind::Simple as i32 == TVMFFIDefRegionKind::kTVMFFIDefRegionKindSimple as i32);
171};
172
173/// Interrupt state of a traversal, an `ffi.VisitInterrupt` object.
174///
175/// Entry points and visitor-layer calls return
176/// `Result<Option<VisitInterrupt>>`: `Ok(None)` means the (sub)graph was
177/// visited completely, `Ok(Some(..))` means a handler halted the traversal
178/// with this interrupt, and `Err` means it failed.
179///
180/// This is an owning object reference, like [`Error`]. Payload reads through
181/// `interrupt.value` borrow the shared [`VisitInterruptObj`]; use [`Self::new`]
182/// instead of a payload-only struct literal. Moving or replacing the payload
183/// through this reference is not supported; clone it to retain a separate owner.
184#[repr(C)]
185#[derive(Clone, ObjectRef)]
186pub struct VisitInterrupt {
187 data: ObjectArc<VisitInterruptObj>,
188}
189
190impl VisitInterrupt {
191 /// Create an interrupt carrying an FFI-compatible payload back to the caller of
192 /// the visit.
193 ///
194 /// # Arguments
195 ///
196 /// * `payload` - The payload, such as the value that was found.
197 ///
198 /// # Returns
199 ///
200 /// The interrupt.
201 #[inline(always)]
202 pub fn new<T: Into<Any>>(payload: T) -> Self {
203 Self::from_object(VisitInterruptObj {
204 base_: Object::new(),
205 value: payload.into(),
206 })
207 }
208}
209
210/// Native state and default traversal for [`StructuralVisitEngine`].
211/// The engine owns allocation, scoped dispatch and results.
212///
213/// # Safety
214///
215/// `Self` must have a complete initialized C layout with [`StructuralVisitorObj`]
216/// at offset zero, matching its registered native type and every field hooks access.
217/// Native-mutated fields require interior mutability; shared references must not
218/// freeze them during recursion. Hooks must preserve the vtable and valid region state.
219///
220/// The base must not implement [`crate::ObjectCore`] or otherwise allow callers
221/// to retain the runtime. Hooks may access the runtime and borrowed traversal
222/// state only synchronously, on the calling thread.
223///
224/// `default_visit_raw` returns one owning ABI None/Error/VisitInterrupt and recurses only
225/// through the installed vtable, without a whole-base mutable reference spanning
226/// recursion. Failures use that result carrier; panics must not cross native callbacks.
227pub unsafe trait StructuralVisitorBase: Sized {
228 /// Initialize the complete base before the engine installs its runtime header.
229 fn new() -> Result<Self>;
230
231 /// Native type registered for this complete base layout.
232 fn type_index() -> i32;
233
234 /// Descend from a borrowed value without dispatching that value again.
235 /// Return one owning ABI None, Error, or VisitInterrupt value.
236 /// Children reenter the installed vtable. The ordinary implementation accesses
237 /// only the visitor prefix, so specialized bases may reuse it.
238 ///
239 /// # Safety
240 /// `runtime` is this live initialized base with a matching installed vtable.
241 /// The input and runtime remain borrowed for this synchronous call. The
242 /// caller has published the current exclusive dispatcher reborrow.
243 unsafe fn default_visit_raw(runtime: *mut Self, value: AnyView<'_>) -> TVMFFIAny;
244}
245
246// SAFETY: this prefix exactly matches the ordinary native visitor. Its mutable
247// region is interior mutable, and it exposes no safe ownership conversion.
248unsafe impl StructuralVisitorBase for StructuralVisitorObj {
249 #[inline(always)]
250 fn new() -> Result<Self> {
251 Ok(Self::default())
252 }
253
254 fn type_index() -> i32 {
255 static TYPE_INDEX: ColdInitLazyLock<i32> =
256 ColdInitLazyLock::new(|| registered_type_index(StructuralVisitorObj::TYPE_KEY));
257 *TYPE_INDEX
258 }
259
260 #[inline(always)]
261 unsafe fn default_visit_raw(runtime: *mut Self, value: AnyView<'_>) -> TVMFFIAny {
262 unsafe { Self::visit_children_raw(runtime, *value.as_raw_ffi_any()) }
263 }
264}
265
266/// Scoped Rust visitor runtime with a statically selected native base.
267///
268/// Enter through [`StructuralVisitEngine::visit_root`]. The engine owns the complete base
269/// and the dispatcher reborrow handle, while the dispatcher stays in the outer
270/// call. Engines and their owning handles are never exposed to callbacks.
271///
272/// Select a custom base with `StructuralVisitEngine::<Base>::visit_root(root, input)`.
273/// Handlers use `StructuralVisitorView<'_, Target, Base>` for recursion and
274/// base-state inspection; normalization preserves the actual target's identity.
275#[repr(C)]
276pub struct StructuralVisitEngine<Base = StructuralVisitorObj> {
277 base: Base,
278 dispatcher: *mut c_void,
279}
280
281/// The running structural visit, passed to each handler.
282///
283/// A handler that matches a node decides which children are visited: call
284/// [`default_visit`](Self::default_visit) to visit every child,
285/// [`visit`](Self::visit) to visit one value, or neither to skip the children.
286/// Pass the handler's own `self` as `dispatch` to every method. A handler
287/// returns `Ok(None)` to continue, or `Ok(Some(interrupt))` to stop the whole
288/// visit with a [`VisitInterrupt`].
289///
290/// See also [`structural_visit`].
291pub struct StructuralVisitorView<'a, D, Base = StructuralVisitorObj> {
292 runtime: StructuralVisitorHandle,
293 #[allow(clippy::type_complexity)]
294 _marker: PhantomData<(&'a mut (), fn(&mut D) -> Base, Rc<()>)>,
295}
296
297// -----------------------------------------------------------------------------
298// Public visitor API and entrypoint
299// -----------------------------------------------------------------------------
300
301impl<D, Base: StructuralVisitorBase> StructuralVisitorView<'_, D, Base> {
302 /// Inspect the base state; the borrow cannot outlive this view.
303 #[inline(always)]
304 pub fn base(&self) -> &Base {
305 // SAFETY: the unsafe base contract permits shared inspection, including
306 // native mutation through interior-mutable fields during recursion.
307 unsafe { &*self.runtime.cast::<Base>() }
308 }
309
310 /// Return the definition region the current value is in.
311 ///
312 /// Reflected fields marked as definitions set it while their values are
313 /// visited; it is [`DefRegionKind::None`] elsewhere.
314 ///
315 /// # Returns
316 ///
317 /// The current definition region.
318 #[inline(always)]
319 pub fn def_region_kind(&self) -> DefRegionKind {
320 // SAFETY: the view's runtime is live.
321 unsafe { std::mem::transmute(*(*self.runtime).def_region_mode.get()) }
322 }
323
324 /// Run `callback` under a scoped definition region.
325 ///
326 /// An enclosing [`DefRegionKind::Pattern`] region keeps precedence; the
327 /// previous region is restored on all exits.
328 ///
329 /// # Arguments
330 ///
331 /// * `kind` - The definition region to enter.
332 /// * `callback` - Runs inside the region, with this view.
333 ///
334 /// # Returns
335 ///
336 /// The result of `callback`.
337 #[inline(always)]
338 pub fn with_def_region_kind<R>(
339 &mut self,
340 kind: DefRegionKind,
341 callback: impl FnOnce(&mut Self) -> R,
342 ) -> R {
343 // SAFETY: the view's runtime outlives the callback.
344 StructuralVisitorObj::with_def_region_kind(self.runtime, kind, || callback(self))
345 }
346
347 /// Visit `value` through the dispatcher, as if the traversal had reached it.
348 ///
349 /// # Arguments
350 ///
351 /// * `dispatch` - The handler's own `self`.
352 /// * `value` - The value to visit.
353 ///
354 /// # Returns
355 ///
356 /// `None` to continue, or the interrupt that stopped the visit.
357 ///
358 /// # Errors
359 ///
360 /// Returns the error of the handler that `value` reaches.
361 #[inline(always)]
362 pub fn visit<'input, T>(&mut self, dispatch: &mut D, value: T) -> Result<Option<VisitInterrupt>>
363 where
364 T: Into<AnyView<'input>>,
365 {
366 let value: AnyView<'input> = value.into();
367 // SAFETY: the view ties this runtime to D; the check establishes the
368 // instance, and publishing uses the caller's actual exclusive reborrow.
369 unsafe {
370 visit_result_from_any(Any::from_raw_ffi_any(
371 StructuralVisitEngine::<Base>::visit_raw(
372 self.runtime,
373 dispatch,
374 *value.as_raw_ffi_any(),
375 ),
376 ))
377 }
378 }
379
380 /// Visit the children of `value` through the dispatcher, without dispatching
381 /// `value` itself again.
382 ///
383 /// Children are found by the type's registered visit hook or its reflected
384 /// fields. A hook uses the visitor only through its arguments, during the
385 /// call, and must not keep it or call it outside them. The visit stops at the first child that returns an interrupt.
386 ///
387 /// # Arguments
388 ///
389 /// * `dispatch` - The handler's own `self`.
390 /// * `value` - The value whose children to visit.
391 ///
392 /// # Returns
393 ///
394 /// `None` to continue, or the interrupt that stopped the visit.
395 ///
396 /// # Errors
397 ///
398 /// Returns the first error of a child's handler.
399 #[inline(always)]
400 pub fn default_visit<'input, T>(
401 &self,
402 dispatch: &mut D,
403 value: T,
404 ) -> Result<Option<VisitInterrupt>>
405 where
406 T: Into<AnyView<'input>>,
407 {
408 // SAFETY: type/layout and scope come from the private view constructor;
409 // the supplied borrow must identify this runtime's dispatcher instance.
410 unsafe {
411 let context = StructuralVisitEngine::<Base>::checked_dispatcher(self.runtime, dispatch);
412 let value: AnyView<'input> = value.into();
413 visit_result_from_any(Any::from_raw_ffi_any(
414 StructuralVisitEngine::<Base>::default_visit_raw(
415 self.runtime,
416 context,
417 *value.as_raw_ffi_any(),
418 ),
419 ))
420 }
421 }
422}
423
424/// Visit `root` with an owned or borrowed [`Dispatcher`](crate::Dispatcher).
425///
426/// Handlers receive [`StructuralVisitorView`] for explicit recursion and keep state on the
427/// dispatcher. Unmatched values use default child traversal. Pass `&mut D` to
428/// keep the dispatcher and inspect its state after the visit. An owned dispatcher
429/// is held in the outer call and uses the same borrowed traversal engine.
430///
431/// Callbacks must report failures through [`Result`] and must not panic: a panic
432/// reaching the non-unwinding native callback boundary aborts the process.
433///
434/// A callback must not modify an `ffi.List` or `ffi.Dict` that holds the
435/// value it receives or one of that value's ancestors: the traversal borrows
436/// their cells while the callback runs.
437///
438/// # Arguments
439///
440/// * `root` - Value borrowed for traversal, convertible into an [`AnyView`].
441/// * `dispatcher` - Owned or mutably borrowed dispatcher implementing [`IntoDispatcher`].
442/// Matching handlers control recursion through [`StructuralVisitorView`];
443/// unmatched values use default child traversal.
444///
445/// # Examples
446///
447/// Sum integer leaves while default traversal descends into the array:
448///
449/// ```
450/// # fn main() -> tvm_ffi::Result<()> {
451/// use tvm_ffi::{dispatch, structural_visit, AnyView, Array, Result, VisitInterrupt};
452/// use tvm_ffi::StructuralVisitorView;
453///
454/// struct Sum(i64);
455/// #[dispatch(visit, (AnyView<'_>, &mut StructuralVisitorView<'_, Self>), Result<Option<VisitInterrupt>>)]
456/// impl Sum {
457/// fn visit_integer(
458/// &mut self,
459/// value: i64,
460/// _: &mut StructuralVisitorView<'_, Self>,
461/// ) -> Result<Option<VisitInterrupt>> {
462/// self.0 += value;
463/// Ok(None)
464/// }
465/// }
466///
467/// let root = Array::from_vec(vec![1_i64, 2, 3]);
468/// let mut sum = Sum(0);
469/// assert!(structural_visit(&root, &mut sum)?.is_none());
470/// assert_eq!(sum.0, 6);
471/// # Ok(())
472/// # }
473/// ```
474pub fn structural_visit<'input, R, H, M, D, O>(
475 root: R,
476 dispatcher: H,
477) -> Result<Option<VisitInterrupt>>
478where
479 R: Into<AnyView<'input>>,
480 O: BorrowMut<D>,
481 for<'v, 'c, 'r> H: IntoDispatcher<
482 (
483 AnyView<'v>,
484 &'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
485 ),
486 Option<Result<Option<VisitInterrupt>>>,
487 M,
488 Target = D,
489 Output = O,
490 >,
491 for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
492 (
493 AnyView<'v>,
494 &'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
495 ),
496 Option<Result<Option<VisitInterrupt>>>,
497 >,
498{
499 StructuralVisitEngine::visit_root(root, dispatcher)
500}
501
502impl super::dispatch::CallbackResult<Result<Option<VisitInterrupt>>>
503 for Result<Option<VisitInterrupt>>
504{
505 #[inline(always)]
506 fn callback_result(self) -> Result<Option<VisitInterrupt>> {
507 self
508 }
509}
510
511// -----------------------------------------------------------------------------
512// StructuralVisitEngine implementation
513// -----------------------------------------------------------------------------
514
515// Generic runtime: only this implementation projects the dispatcher after Base.
516impl<Base: StructuralVisitorBase> StructuralVisitEngine<Base> {
517 /// Visit `root` with `dispatcher` on the native base `Base`. See also [`structural_visit`].
518 #[inline(always)]
519 pub fn visit_root<'input, R, H, M, D, O>(
520 root: R,
521 dispatcher: H,
522 ) -> Result<Option<VisitInterrupt>>
523 where
524 R: Into<AnyView<'input>>,
525 O: BorrowMut<D>,
526 for<'v, 'c, 'r> H: IntoDispatcher<
527 (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
528 Option<Result<Option<VisitInterrupt>>>,
529 M,
530 Target = D,
531 Output = O,
532 >,
533 for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
534 (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
535 Option<Result<Option<VisitInterrupt>>>,
536 >,
537 {
538 // Normalize the input once. Named dispatchers keep their own recursive view
539 // type; callback and tuple targets are their aggregate adapters.
540 let mut storage = dispatcher.into_dispatcher();
541 let root = root.into();
542 Self::run_visitor::<D>(*root.as_raw_ffi_any(), storage.borrow_mut())
543 }
544
545 /// The engine whose first field is the native prefix `runtime`.
546 ///
547 /// # Safety
548 /// `runtime` must belong to the live, synchronous Rust traversal that created
549 /// this exact repr(C) engine. An arbitrary native visitor is not such an engine.
550 #[inline(always)]
551 unsafe fn from_handle(runtime: StructuralVisitorHandle) -> *mut Self {
552 runtime.cast()
553 }
554
555 /// Publish the current exclusive `dispatch` reborrow and visit borrowed `raw`.
556 /// Return one owning raw None/interrupt/Error token.
557 ///
558 /// # Safety
559 /// `runtime` must belong to this live `D` invocation and `raw` remain valid
560 /// through recursion; the caller cannot use competing dispatcher borrows.
561 #[inline(always)]
562 unsafe fn visit_raw<D>(
563 runtime: StructuralVisitorHandle,
564 dispatch: &mut D,
565 raw: TVMFFIAny,
566 ) -> TVMFFIAny {
567 let context = unsafe { Self::checked_dispatcher(runtime, dispatch) };
568 unsafe { (*Self::from_handle(runtime)).dispatcher = context };
569 StructuralVisitorObj::call_visitor_raw(runtime, raw)
570 }
571
572 /// Check runtime identity and return the current `dispatch` reborrow pointer.
573 ///
574 /// # Safety
575 /// `runtime` must be the live engine for `D`; address equality does not establish
576 /// type or lifetime validity. Mismatch panics before a pointer is published.
577 // The caller has already established matching runtime/D types and scope.
578 // Read only the stored raw address: do not create a competing mutable borrow.
579 #[inline(always)]
580 unsafe fn checked_dispatcher<D>(
581 runtime: StructuralVisitorHandle,
582 dispatch: &mut D,
583 ) -> *mut c_void {
584 let current = dispatcher_ptr(dispatch);
585 if current != unsafe { (*Self::from_handle(runtime)).dispatcher } {
586 core::hint::cold_path();
587 wrong_dispatcher();
588 }
589 current
590 }
591
592 /// Build the native visitor callback for concrete dispatcher `D`. Its higher-ranked
593 /// view arguments restrict all borrows to the synchronous handler invocation.
594 #[inline(always)]
595 fn vtable<D>() -> &'static StructuralVisitorVTable
596 where
597 for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
598 (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
599 Option<Result<Option<VisitInterrupt>>>,
600 >,
601 {
602 &StructuralVisitorVTable {
603 visit: Self::rust_vtable_visit_raw::<D>,
604 }
605 }
606
607 /// Enter a native visitor callback with borrowed `value`; return one owning token.
608 /// Restore the incoming dispatcher provenance before returning to the caller.
609 ///
610 /// # Safety
611 /// `runtime` must have this `D` vtable and a live published reborrow. The erased
612 /// ABI lifetime on `value` is valid only for this call. Callback failures use
613 /// Result; Rust panics cannot cross this native C boundary.
614 unsafe extern "C" fn rust_vtable_visit_raw<D>(
615 runtime: StructuralVisitorHandle,
616 value: AnyView<'static>,
617 ) -> TVMFFIAny
618 where
619 for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
620 (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
621 Option<Result<Option<VisitInterrupt>>>,
622 >,
623 {
624 let context = unsafe { (*Self::from_handle(runtime)).dispatcher };
625 let raw = *value.as_raw_ffi_any();
626 let result = if raw.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
627 core::hint::cold_path();
628 TVMFFIAny::new()
629 } else {
630 let dispatch = unsafe { &mut *context.cast::<D>() };
631 let mut visitor = unsafe { Self::visitor_context::<D>(runtime) };
632 // SAFETY: the native caller keeps the input alive for this synchronous
633 // dispatch, and the handler does not modify a list or dict cell
634 // holding it (the callback contract); the higher-ranked callback
635 // cannot retain its borrowed view.
636 match unsafe {
637 crate::Dispatcher::dispatch_tuple(
638 &mut *dispatch,
639 (AnyView::from_raw_ffi_any(raw), &mut visitor),
640 )
641 } {
642 Some(result) => visit_result_raw(result, raw),
643 None => unsafe { Self::default_visit_raw(runtime, dispatcher_ptr(dispatch), raw) },
644 }
645 };
646 // The enclosing callback may resume using its ancestor mutable borrow.
647 // Native hooks cannot restore Rust-only borrow provenance for us.
648 unsafe { (*Self::from_handle(runtime)).dispatcher = context };
649 result
650 }
651
652 /// Create the handler view for borrowed `runtime`, without retaining it.
653 ///
654 /// # Safety
655 /// The live runtime's vtable must match D. Keep this view within the current
656 /// higher-ranked handler call, never outliving the runtime or dispatcher.
657 #[inline(always)]
658 unsafe fn visitor_context<'a, D>(
659 runtime: StructuralVisitorHandle,
660 ) -> StructuralVisitorView<'a, D, Base> {
661 StructuralVisitorView {
662 runtime,
663 _marker: PhantomData,
664 }
665 }
666
667 /// Publish `context` and descend through borrowed `raw`, then annotate errors.
668 /// The raw return transfers one owning None/interrupt/Error token.
669 ///
670 /// # Safety
671 /// `runtime` must be live and `context` its current exclusive dispatcher reborrow;
672 /// `raw` remains borrowed and must survive the complete synchronous descent.
673 #[inline(always)]
674 unsafe fn default_visit_raw(
675 runtime: StructuralVisitorHandle,
676 context: *mut c_void,
677 raw: TVMFFIAny,
678 ) -> TVMFFIAny {
679 // Context is the supplied exclusive reborrow. Publish it before descent;
680 // the handle-only scanner never creates or uses a competing D reference.
681 unsafe { (*Self::from_handle(runtime)).dispatcher = context };
682 unsafe {
683 StructuralVisitorObj::with_visit_context_raw(
684 Base::default_visit_raw(runtime.cast(), AnyView::from_raw_ffi_any(raw)),
685 raw,
686 )
687 }
688 }
689
690 /// Visit borrowed `root` with `dispatch` on a fresh engine; the root's owner outlives it.
691 pub(super) fn run_visitor<D>(
692 root: TVMFFIAny,
693 dispatch: &mut D,
694 ) -> Result<Option<VisitInterrupt>>
695 where
696 for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
697 (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
698 Option<Result<Option<VisitInterrupt>>>,
699 >,
700 {
701 let base = Base::new()?;
702 let type_index = Base::type_index();
703 // The managed runtime may borrow its dispatcher only for this call.
704 // Trusted hooks release all temporary handles before the owner is dropped.
705 let mut engine = unsafe {
706 ObjectArc::<Object>::new_opaque_unchecked(
707 StructuralVisitEngine {
708 base,
709 dispatcher: dispatcher_ptr(dispatch),
710 },
711 type_index,
712 )
713 };
714 let runtime = ObjectArc::as_mut_ptr(&mut engine).cast::<StructuralVisitorObj>();
715 // The engine and outer dispatcher storage stay stationary for the whole run.
716 // Only raw field projections cross recursion; no whole-engine borrow survives.
717 unsafe {
718 (*runtime).vtable = Self::vtable::<D>();
719 *(*runtime).def_region_mode.get() = DefRegionKind::None as i32;
720 }
721 // Keep the result owned while releasing the runtime, including on unwind.
722 let result =
723 unsafe { Any::from_raw_ffi_any(StructuralVisitorObj::call_visitor_raw(runtime, root)) };
724 // Trusted hooks have released every temporary strong/weak runtime reference.
725 drop(engine);
726 // The public traversal boundary, as the C++ `StructuralVisitExpected`.
727 visit_result_from_any(result).map_err(|error| enrich_visit_error(error, root))
728 }
729}
730
731// -----------------------------------------------------------------------------
732// Prefix-only default traversal and native ABI operations
733// -----------------------------------------------------------------------------
734
735impl StructuralVisitorObj {
736 /// Consume `result`, naming the borrowed `value` in it when it is an Error.
737 #[inline(always)]
738 fn with_visit_context_raw(result: TVMFFIAny, value: TVMFFIAny) -> TVMFFIAny {
739 if result.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
740 result
741 } else {
742 Self::with_visit_stop_context_raw(result, value)
743 }
744 }
745
746 // Continuing visits need no owning conversion or input inspection. Keep
747 // interrupt/error handling out of their recursive frames.
748 #[cold]
749 #[inline(never)]
750 extern "C-unwind" fn with_visit_stop_context_raw(
751 result: TVMFFIAny,
752 value: TVMFFIAny,
753 ) -> TVMFFIAny {
754 if result.type_index != TVMFFITypeIndex::kTVMFFIError as i32
755 || value.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32
756 {
757 result
758 } else {
759 // SAFETY: the Error tag establishes the owned object's representation.
760 unsafe {
761 let error = take_error(Any::from_raw_ffi_any(result));
762 Any::into_raw_ffi_any(update_visit_error_context(error, value).into())
763 }
764 }
765 }
766
767 /// Visit the children of `value` through its hook or reflected fields, without
768 /// dispatching `value` itself.
769 ///
770 /// # Safety
771 ///
772 /// `runtime` is live with its dispatcher published, and `value` outlives descent.
773 #[inline(always)]
774 unsafe fn visit_children_raw(runtime: StructuralVisitorHandle, value: TVMFFIAny) -> TVMFFIAny {
775 if let Some(attr) = structural_visit_column().get_raw_ffi_any(value.type_index)
776 && attr.type_index != TVMFFITypeIndex::kTVMFFINone as i32
777 {
778 return unsafe { Self::call_structural_visit_hook_raw(runtime, value, attr) };
779 }
780
781 if value.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
782 TVMFFIAny::new()
783 } else {
784 unsafe { Self::visit_reflected_fields_raw(runtime, value) }
785 }
786 }
787
788 /// Visit each reflected field of the object in `value`.
789 ///
790 /// # Safety
791 ///
792 /// As [`Self::visit_children_raw`]. Outlined (frequent, not cold) to keep reflection
793 /// scratch out of recursive frames.
794 #[inline(never)]
795 unsafe extern "C-unwind" fn visit_reflected_fields_raw(
796 runtime: StructuralVisitorHandle,
797 value: TVMFFIAny,
798 ) -> TVMFFIAny {
799 let type_info = unsafe { TVMFFIGetTypeInfo(value.type_index) };
800 if type_info.is_null() {
801 return Any::into_raw_ffi_any(unregistered_visit_type_error(value.type_index).into());
802 }
803 let def_region_kind = unsafe {
804 free_var_child_region(
805 std::mem::transmute::<i32, DefRegionKind>(*(*runtime).def_region_mode.get()),
806 seq_hash_kind(type_info),
807 )
808 };
809 let object = unsafe { value.data_union.v_obj } as *mut u8;
810 let halted = unsafe {
811 for_each_field_info(type_info, &mut |field| {
812 let result = Self::visit_reflected_field(runtime, object, field, def_region_kind);
813 if result.is_none() {
814 ControlFlow::Continue(())
815 } else {
816 ControlFlow::Break(result)
817 }
818 })
819 };
820 Any::into_raw_ffi_any(halted.unwrap_or_else(Any::new))
821 }
822
823 /// Visit one reflected `field` of `object` under `inherited_region`.
824 ///
825 /// # Safety
826 ///
827 /// `runtime` and `object` are live, and `field` describes `object`'s storage.
828 unsafe fn visit_reflected_field(
829 runtime: StructuralVisitorHandle,
830 object: *mut u8,
831 field: &TVMFFIFieldInfo,
832 inherited_region: DefRegionKind,
833 ) -> Any {
834 if field.flags & FLAG_SEQ_HASH_IGNORE != 0 {
835 return Any::new();
836 }
837
838 let Some(getter) = field.getter else {
839 return Any::from(unsafe {
840 missing_field_getter_error("visitor", field.name.as_str())
841 });
842 };
843 let address = unsafe { object.offset(field.offset as isize) as *mut c_void };
844 // Own the getter result so partial writes and recursive borrows drop safely.
845 let mut child = Any::new();
846 if unsafe { getter(address, Any::as_raw_ffi_any_mut(&mut child)) } != 0 {
847 return Any::from(Error::from_raised());
848 }
849
850 let borrowed = AnyView::from(&child);
851 let child_region = field_def_region(field, inherited_region);
852 Self::with_def_region_kind(runtime, child_region, || unsafe {
853 Any::from_raw_ffi_any(Self::call_visitor_raw(runtime, *borrowed.as_raw_ffi_any()))
854 })
855 }
856
857 /// Dispatch borrowed `value` through the runtime's vtable.
858 #[inline(always)]
859 fn call_visitor_raw(runtime: StructuralVisitorHandle, value: TVMFFIAny) -> TVMFFIAny {
860 if value.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
861 core::hint::cold_path();
862 return TVMFFIAny::new();
863 }
864 unsafe {
865 let callback = (*(*runtime).vtable).visit;
866 callback(runtime, AnyView::from_raw_ffi_any(value))
867 }
868 }
869
870 /// Call the registered visit `hook` (native pointer or packed function) on `value`.
871 ///
872 /// # Safety
873 ///
874 /// `runtime` is live with its dispatcher published, `value` and `hook` outlive the
875 /// call, and a native hook has the visit ABI for this runtime's layout and obeys
876 /// [`StructuralVisitorBase`]'s non-escaping contract.
877 #[inline(always)]
878 pub unsafe fn call_structural_visit_hook_raw(
879 runtime: *mut Self,
880 value: TVMFFIAny,
881 hook: TVMFFIAny,
882 ) -> TVMFFIAny {
883 unsafe {
884 if hook.type_index == TVMFFITypeIndex::kTVMFFIOpaquePtr as i32 {
885 let pointer = hook.data_union.v_ptr;
886 // Native pointer registration is an unsafe contract: a non-null
887 // callback with this ABI, borrowed runtime lifetime and owning result.
888 let hook: FStructuralVisit = std::mem::transmute(pointer);
889 hook(runtime, AnyView::from_raw_ffi_any(value))
890 } else {
891 core::hint::cold_path();
892 Self::call_structural_visit_hook_slow_raw(runtime, value, hook)
893 }
894 }
895 }
896
897 /// The packed-function arm of [`Self::call_structural_visit_hook_raw`], outlined as cold;
898 /// its result tag is checked.
899 #[cold]
900 #[inline(never)]
901 extern "C-unwind" fn call_structural_visit_hook_slow_raw(
902 runtime: StructuralVisitorHandle,
903 value: TVMFFIAny,
904 hook: TVMFFIAny,
905 ) -> TVMFFIAny {
906 // Packed hooks (including Python class methods) can return any well-formed
907 // Any through safe APIs. Their result tag needs validation before decoding.
908 let result = unsafe {
909 Function::try_from(AnyView::from_raw_ffi_any(hook)).and_then(|function| {
910 // Borrow the live runtime for this synchronous hook; it may not retain it.
911 let object = runtime.cast::<TVMFFIObject>();
912 let mut runtime = TVMFFIAny::new();
913 runtime.type_index = (*object).type_index;
914 runtime.data_union.v_obj = object;
915 function.call_packed(&[
916 AnyView::from_raw_ffi_any(runtime),
917 AnyView::from_raw_ffi_any(value),
918 ])
919 })
920 };
921 let result = match result {
922 Ok(value) => checked_hook_result(value),
923 Err(error) => error.into(),
924 };
925 Any::into_raw_ffi_any(result)
926 }
927
928 /// Run `callback` under requested `kind` on live borrowed `visitor`, preserving
929 /// Pattern precedence. Restore prior region on return or Rust unwind. Internal
930 /// callers keep the runtime live and retain no reference to its region field.
931 #[inline(always)]
932 fn with_def_region_kind<T>(
933 visitor: StructuralVisitorHandle,
934 kind: DefRegionKind,
935 callback: impl FnOnce() -> T,
936 ) -> T {
937 unsafe {
938 let previous = *(*visitor).def_region_mode.get();
939 if previous == kind as i32 || previous == DefRegionKind::Pattern as i32 {
940 // Inherited descent does not change state; Pattern keeps precedence.
941 return callback();
942 }
943 *(*visitor).def_region_mode.get() = kind as i32;
944 struct Restore {
945 visitor: StructuralVisitorHandle,
946 previous: i32,
947 }
948 impl Drop for Restore {
949 #[inline(always)]
950 fn drop(&mut self) {
951 unsafe { *(*self.visitor).def_region_mode.get() = self.previous };
952 }
953 }
954 let _restore = Restore { visitor, previous };
955 callback()
956 }
957 }
958}
959
960// -----------------------------------------------------------------------------
961// Private utility functions
962// -----------------------------------------------------------------------------
963
964#[inline(always)]
965fn dispatcher_ptr<D>(dispatch: &mut D) -> *mut c_void {
966 std::ptr::from_mut(dispatch).cast()
967}
968
969// The visit result carrier, as C++: None continues; a VisitInterrupt or an Error stops.
970#[inline(always)]
971fn visit_result_raw(result: Result<Option<VisitInterrupt>>, raw: TVMFFIAny) -> TVMFFIAny {
972 match result {
973 Ok(None) => TVMFFIAny::new(),
974 Ok(Some(interrupt)) => Any::into_raw_ffi_any(interrupt.into()),
975 Err(error) => {
976 core::hint::cold_path();
977 StructuralVisitorObj::with_visit_context_raw(Any::into_raw_ffi_any(error.into()), raw)
978 }
979 }
980}
981
982#[inline(always)]
983pub(super) fn visit_result_from_any(result: Any) -> Result<Option<VisitInterrupt>> {
984 let tag = result.type_index();
985 if tag == TVMFFITypeIndex::kTVMFFINone as i32 {
986 return Ok(None);
987 }
988 core::hint::cold_path();
989 if tag == TVMFFITypeIndex::kTVMFFIError as i32 {
990 // SAFETY: internal results obey the documented owning carrier protocol.
991 Err(unsafe { take_error(result) })
992 } else {
993 // SAFETY: the owning result protocol established a non-null interrupt.
994 // Transfer that same reference, including carriers shared by native hooks.
995 use crate::type_traits::AnyCompatible;
996 let mut raw = Any::into_raw_ffi_any(result);
997 Ok(Some(unsafe {
998 VisitInterrupt::move_from_any_after_check(&mut raw)
999 }))
1000 }
1001}
1002
1003// A packed hook's owning Any is valid, but its result type is dynamic.
1004#[inline(always)]
1005fn checked_hook_result(value: Any) -> Any {
1006 let tag = value.type_index();
1007 if tag == TVMFFITypeIndex::kTVMFFINone as i32
1008 || tag == TVMFFITypeIndex::kTVMFFIError as i32
1009 || tag == TVMFFITypeIndex::kTVMFFIVisitInterrupt as i32
1010 {
1011 value
1012 } else {
1013 invalid_visit_hook_result_error().into()
1014 }
1015}
1016
1017/// Consume exactly one Error reference after its integer tag was checked.
1018#[inline(always)]
1019unsafe fn take_error(value: Any) -> Error {
1020 use crate::type_traits::AnyCompatible;
1021 let mut raw = Any::into_raw_ffi_any(value);
1022 unsafe { Error::move_from_any_after_check(&mut raw) }
1023}
1024
1025// Columns are process-lifetime registry entries; a missing per-type cell still
1026// falls back to reflected traversal. A missing required column is an error.
1027static STRUCTURAL_VISIT_COLUMN: ColdInitLazyLock<TypeAttrColumn> = ColdInitLazyLock::new(|| {
1028 TypeAttrColumn::lookup(STRUCTURAL_VISIT_ATTR).expect("missing __s_visit__ column")
1029});
1030
1031/// Return the shared visit-hook column without caching optional per-type cells.
1032#[inline(always)]
1033fn structural_visit_column() -> TypeAttrColumn {
1034 *STRUCTURAL_VISIT_COLUMN
1035}
1036
1037// -----------------------------------------------------------------------------
1038// Outlined errors
1039// -----------------------------------------------------------------------------
1040
1041#[cold]
1042#[inline(never)]
1043fn unregistered_visit_type_error(type_index: i32) -> Error {
1044 runtime_error(&format!(
1045 "native visitor: unregistered type index {type_index}"
1046 ))
1047}
1048
1049#[cold]
1050#[inline(never)]
1051fn invalid_visit_hook_result_error() -> Error {
1052 Error::new(
1053 TYPE_ERROR,
1054 "structural visit hook must return None or ffi.VisitInterrupt",
1055 )
1056}