vox-postcard 0.8.2

// r[impl schema.translation.skip-unknown]
// r[impl schema.translation.reorder]
// r[impl schema.translation.enum]
// r[impl schema.translation.enum.unknown-variant]
// r[impl schema.errors.unknown-variant-runtime]
/// Compact execution IR for postcard decode.
///
/// This IR sits between `TranslationPlan` (semantic) and Cranelift codegen
/// (machine). It is intended to be:
///
/// - fully concrete (no shape reflection at interpret time)
/// - small (flat instruction sequences with explicit operands)
/// - safe to interpret from a pure-Rust fallback path
///
/// The IR does NOT mention `Peek`, `Partial`, or any facet_reflect primitive.
/// All layout knowledge is baked in at lowering time.
use std::collections::{HashMap, HashSet};
use std::sync::OnceLock;

use facet_core::{EnumRepr, Facet, ScalarType, Shape, Type, UserType};
use vox_jit_cal::{BorrowMode, CalibrationRegistry, DescriptorHandle};
use vox_schema::{SchemaKind, SchemaRegistry};

use crate::error::DeserializeError;
use crate::plan::{FieldOp, TranslationPlan};

/// Cached `VOX_JIT_TRACE_LOWER` env-var lookup. Set the variable to any
/// non-empty value to print one stderr line per cycle-detector hit (decode
/// and encode side), so you can find unexpected `CallSelf` / `SlowPath`
/// emissions when staring at a flame graph.
fn trace_lower_enabled() -> bool {
    static FLAG: OnceLock<bool> = OnceLock::new();
    *FLAG.get_or_init(|| {
        std::env::var_os("VOX_JIT_TRACE_LOWER")
            .map(|v| !v.is_empty())
            .unwrap_or(false)
    })
}

fn trace_cycle_emission(
    ir: &str,
    op: &str,
    shape: &'static Shape,
    top_shape: Option<&'static Shape>,
    in_progress: &HashSet<&'static Shape>,
) {
    if !trace_lower_enabled() {
        return;
    }
    let top = top_shape
        .map(|s| format!("{s}"))
        .unwrap_or_else(|| "<none>".to_owned());
    let in_progress_list = in_progress
        .iter()
        .map(|s| format!("{s}"))
        .collect::<Vec<_>>()
        .join(", ");
    eprintln!(
        "[vox-jit-trace] ir={ir} op={op} shape@{:p}={shape} top={top} in_progress=[{in_progress_list}]",
        shape as *const Shape,
    );
}

// ---------------------------------------------------------------------------
// Wire-level vocabulary
// ---------------------------------------------------------------------------
//
// `OpaqueDescriptorId`, `WirePrimitive`, and `TagWidth` are pure-data IR
// primitives shared with every codec backend; they live in `vox-jit-abi`.
// We re-export them here so existing `vox_postcard::ir::WirePrimitive`
// imports keep working. The facet-aware constructors below (which only
// make sense for the Rust backend) live with the lowerer.

pub use vox_jit_abi::wire::{OpaqueDescriptorId, TagWidth, WirePrimitive};

/// Convert a facet `ScalarType` into the layout-agnostic [`WirePrimitive`]
/// vocabulary. Lives here (not in `vox-jit-abi`) because it depends on
/// facet, which the Swift codec backend does not.
pub fn wire_primitive_from_scalar(s: ScalarType) -> Option<WirePrimitive> {
    Some(match s {
        ScalarType::Unit => WirePrimitive::Unit,
        ScalarType::Bool => WirePrimitive::Bool,
        ScalarType::U8 => WirePrimitive::U8,
        ScalarType::U16 => WirePrimitive::U16,
        ScalarType::U32 => WirePrimitive::U32,
        ScalarType::U64 => WirePrimitive::U64,
        ScalarType::U128 => WirePrimitive::U128,
        ScalarType::USize => WirePrimitive::USize,
        ScalarType::I8 => WirePrimitive::I8,
        ScalarType::I16 => WirePrimitive::I16,
        ScalarType::I32 => WirePrimitive::I32,
        ScalarType::I64 => WirePrimitive::I64,
        ScalarType::I128 => WirePrimitive::I128,
        ScalarType::ISize => WirePrimitive::ISize,
        ScalarType::F32 => WirePrimitive::F32,
        ScalarType::F64 => WirePrimitive::F64,
        ScalarType::String => WirePrimitive::String,
        ScalarType::Str => WirePrimitive::String,
        ScalarType::CowStr => WirePrimitive::String,
        ScalarType::Char => WirePrimitive::Char,
        _ => return None,
    })
}

/// Convert a facet `EnumRepr` into the layout-agnostic [`TagWidth`]
/// vocabulary, returning `None` for representations whose tag width is not
/// statically known (Rust default repr).
pub fn tag_width_from_enum_repr(repr: EnumRepr) -> Option<TagWidth> {
    match repr {
        EnumRepr::U8 | EnumRepr::I8 => Some(TagWidth::U8),
        EnumRepr::U16 | EnumRepr::I16 => Some(TagWidth::U16),
        EnumRepr::U32 | EnumRepr::I32 => Some(TagWidth::U32),
        EnumRepr::U64 | EnumRepr::I64 | EnumRepr::USize | EnumRepr::ISize => Some(TagWidth::U64),
        EnumRepr::Rust | EnumRepr::RustNPO => None,
    }
}

// ---------------------------------------------------------------------------
// IR instruction set
// ---------------------------------------------------------------------------

/// A single IR instruction for the decode path.
///
/// Instructions operate on an implicit cursor (input bytes) and write to
/// an implicit destination pointer (`*mut u8` base + field offsets). The
/// interpreter tracks both.
///
/// Operand convention:
///   - `dst_offset`: byte offset from the base of the struct being written
///   - `block_id`:   index into `DecodeProgram::blocks`
#[derive(Debug, Clone)]
pub enum DecodeOp {
    // -----------------------------------------------------------------------
    // Primitive reads
    // -----------------------------------------------------------------------
    /// Read a scalar primitive from the cursor and write it to `dst_offset`
    /// in the current destination.
    ReadScalar {
        prim: WirePrimitive,
        dst_offset: usize,
    },

    /// Read a varint length prefix followed by `len * elem_size` raw bytes
    /// and copy them directly into a `Vec<T>` at `dst_offset` (using the
    /// calibrated descriptor for allocation). Used for element types whose
    /// postcard wire format is bit-identical to the in-memory representation:
    /// `u8`/`i8` (any endian, 1 byte), `f32`/`f64` (postcard-spec'd as fixed
    /// little-endian, so identical on LE hosts), `bool` (1 byte, validated to
    /// be 0 or 1). Replaces the per-element decode loop with a single bulk
    /// memcpy plus, for bools, a vectorizable bitwise-OR scan to reject any
    /// byte > 1.
    ReadFixedVec {
        dst_offset: usize,
        descriptor: OpaqueDescriptorId,
        elem_size: usize,
        /// When true, after the memcpy validate every byte is 0 or 1 and
        /// fail with an `InvalidValue` error otherwise. Required for
        /// `Vec<bool>` since postcard rejects bytes >= 2.
        validate_bool: bool,
    },

    /// Read a varint-length-prefixed UTF-8 string into `String` at
    /// `dst_offset` (uses opaque descriptor for allocation).
    ReadString {
        dst_offset: usize,
        descriptor: OpaqueDescriptorId,
    },

    /// Read a varint-length-prefixed UTF-8 string into `Cow<str>` at
    /// `dst_offset`.
    ReadCowStr { dst_offset: usize, borrowed: bool },

    /// Read a varint-length-prefixed UTF-8 string into `&str` at `dst_offset`.
    ReadStrRef { dst_offset: usize },

    /// Read a varint-length-prefixed byte slice and initialize `Cow<[u8]>`
    /// at `dst_offset`.
    ReadCowByteSlice { dst_offset: usize, borrowed: bool },

    /// Read a varint-length-prefixed byte slice and initialize `&[u8]`
    /// at `dst_offset`.
    ReadByteSliceRef { dst_offset: usize },

    /// Read a u32le-length-prefixed opaque payload and initialize the target
    /// via the shape's opaque adapter.
    ReadOpaque {
        shape: &'static Shape,
        dst_offset: usize,
    },

    // -----------------------------------------------------------------------
    // Skip operations (remote fields absent in local type)
    // -----------------------------------------------------------------------

    // r[impl schema.translation.skip-unknown]
    /// Skip one postcard value described by a pre-resolved schema kind.
    /// The kind is stored inline so the interpreter does not need a registry.
    SkipValue { kind: SchemaKind },

    // r[impl schema.translation.fill-defaults]
    /// Initialize a local field via its `Default` implementation. Emitted for
    /// every local struct field that has no corresponding remote field on the
    /// wire (schema evolution: remote dropped a field that has a default on
    /// the local side).
    WriteDefault {
        shape: &'static Shape,
        dst_offset: usize,
    },

    // -----------------------------------------------------------------------
    // Option handling
    // -----------------------------------------------------------------------
    /// Decode an `Option<T>` in-place.
    ///
    /// Reads the tag byte (0 = None, 1 = Some).
    ///
    /// - On None: calls `init_none` via vtable to write the None representation
    ///   into `dst_offset`. Does NOT branch; execution continues inline.
    /// - On Some: calls `init_some_and_get_inner` to write the Some tag and
    ///   obtain the pointer to the inner value slot, then jumps to `some_block`
    ///   with the inner pointer as the new base.
    ///
    /// `none_init_fn` and `some_init_fn` are the vtable function pointers
    /// captured at lowering time so the Cranelift backend can embed them as
    /// immediate constants.
    ///
    /// `inner_offset` is the byte offset from the `Option` base to the inner
    /// value slot (determined by probing the vtable at lowering time). Used by
    /// the interpreter to compute the write address directly.
    DecodeOption {
        dst_offset: usize,
        inner_offset: usize,
        some_block: usize,
        /// Exact bytes of a calibrated `None` value.
        none_bytes: Box<[u8]>,
        /// Exact bytes of a calibrated `Some(_)` value before the payload is
        /// overwritten by the inner decode.
        some_bytes: Box<[u8]>,
    },

    /// Decode a `Result<T, E>` in-place.
    ///
    /// Reads the postcard variant index (`0 = Ok`, `1 = Err`) as a varint,
    /// decodes the selected inner value into a temporary buffer, then moves it
    /// into the destination result via the result vtable.
    DecodeResult {
        dst_offset: usize,
        ok_block: usize,
        err_block: usize,
        ok_offset: usize,
        err_offset: usize,
        /// Exact bytes of a calibrated `Ok(_)` value before the payload is
        /// overwritten by the inner decode.
        ok_bytes: Box<[u8]>,
        /// Exact bytes of a calibrated `Err(_)` value before the payload is
        /// overwritten by the inner decode.
        err_bytes: Box<[u8]>,
    },

    /// Decode a `Result<T, E>` via payload scratch storage, then initialize the
    /// destination with the result vtable. This is used when payload types do
    /// not provide defaults, so calibrated direct-write templates cannot be
    /// built safely.
    DecodeResultInit {
        dst_offset: usize,
        ok_block: usize,
        err_block: usize,
        ok_size: usize,
        ok_align: usize,
        err_size: usize,
        err_align: usize,
        init_ok_fn: facet_core::ResultInitOkFn,
        init_err_fn: facet_core::ResultInitErrFn,
    },

    // -----------------------------------------------------------------------
    // Enum handling
    // -----------------------------------------------------------------------
    /// Read the varint enum discriminant from the wire.
    ReadDiscriminant,

    // r[impl schema.translation.enum]
    // r[impl schema.translation.enum.unknown-variant]
    /// Map remote discriminant (held in the interpreter's scratch register)
    /// to a local variant index.  `variant_table[remote_disc]` is the local
    /// index, or `None` for unknown variants (runtime error).
    ///
    /// On match: writes the local discriminant tag bytes to `tag_offset` and
    /// jumps to the block for that variant.
    BranchOnVariant {
        tag_offset: usize,
        tag_width: TagWidth,
        /// `variant_table[remote_index] = Some(local_index) or None`
        variant_table: Vec<Option<usize>>,
        /// per-variant: (local_discriminant_value, block_id)
        variant_blocks: Vec<(u64, usize)>,
    },

    // -----------------------------------------------------------------------
    // Struct / tuple field handling
    // -----------------------------------------------------------------------
    /// Push a new stack frame. The new frame's base pointer is
    /// `current_base + field_offset`, size is `frame_size`.
    /// After `PopFrame` the interpreter returns to the parent base.
    PushFrame {
        field_offset: usize,
        frame_size: usize,
    },

    /// Return to the parent frame (mirrors `PushFrame`).
    PopFrame,

    // -----------------------------------------------------------------------
    // List / array handling
    // -----------------------------------------------------------------------
    /// Read the varint element count. Branch to `empty_block` if zero,
    /// otherwise call `alloc_block` to allocate backing storage then execute
    /// `body_block` for each element.
    ReadListLen {
        descriptor: OpaqueDescriptorId,
        dst_offset: usize,
        empty_block: usize,
        body_block: usize,
    },

    /// Commit the current element count to the `len` field of the list
    /// backing at `dst_offset`. Called after each element is successfully
    /// decoded.
    CommitListLen {
        dst_offset: usize,
        descriptor: OpaqueDescriptorId,
    },

    /// Decode a fixed-count array (length known at lowering time).
    /// Repeats `body_block` exactly `count` times, advancing by `elem_size`
    /// each iteration.
    DecodeArray {
        dst_offset: usize,
        count: usize,
        elem_size: usize,
        body_block: usize,
    },

    // -----------------------------------------------------------------------
    // Opaque fast-path
    // -----------------------------------------------------------------------
    /// Copy the calibrated empty-value bytes for an opaque type into the
    /// destination at `dst_offset`. Used for zero-length lists and strings.
    MaterializeEmpty {
        dst_offset: usize,
        descriptor: OpaqueDescriptorId,
    },

    /// Allocate backing storage for a Vec/String and write the fat-pointer
    /// fields to `dst_offset`. The capacity is in the interpreter's len
    /// register (set by `ReadListLen`).
    ///
    /// `body_block` contains the element decode ops (base = element pointer).
    /// `elem_size` is the stride between elements in the backing allocation.
    /// The Cranelift backend emits the element loop inline using these fields;
    /// the IR interpreter ignores them (it falls back via SlowPath for lists).
    AllocBacking {
        dst_offset: usize,
        descriptor: OpaqueDescriptorId,
        /// IR block containing element decode ops (dst_offset=0, base=elem ptr).
        body_block: usize,
        /// Byte stride between elements in backing storage.
        elem_size: usize,
    },

    /// Allocate a single heap slot for a `Box<T>` and decode the pointee into it.
    ///
    /// Calls `vox_jit_box_alloc(desc, container_ptr)` to allocate and write the
    /// pointer into `dst_offset`. Then decodes the inner type using `body_block`
    /// with the allocated pointer as the new base (`dst_offset=0`).
    AllocBoxed {
        dst_offset: usize,
        descriptor: OpaqueDescriptorId,
        /// IR block containing ops to decode the pointee (dst_offset=0, base=alloc_ptr).
        body_block: usize,
    },

    // -----------------------------------------------------------------------
    // Map handling
    // -----------------------------------------------------------------------
    /// Decode a map (`BTreeMap`/`HashMap`) via the slab strategy.
    ///
    /// Reads the varint entry count, decodes every `(K, V)` pair natively into
    /// a contiguous scratch slab, then hands the slab to facet's
    /// `from_pair_slice` constructor in one call — one boundary crossing for
    /// the whole map, no per-entry vtable dispatch. The JIT win is captured on
    /// the native K/V value decode; the collection assembly is amortized.
    ///
    /// `body_block` decodes one pair into the current slab slot: the key at
    /// offset 0, the value at `value_offset`. The slot base advances by
    /// `pair_stride` bytes per iteration. The slab holds `count` slots of
    /// `pair_stride` bytes, aligned to `pair_align`.
    DecodeMap {
        dst_offset: usize,
        /// facet `MapVTable::from_pair_slice` — builds the map from the slab
        /// (`collect()` into the target collection, with capacity known).
        from_pair_slice: facet_core::MapFromPairSliceFn,
        /// `size_of::<(K, V)>()` — stride between slab slots (facet `pair_stride`).
        pair_stride: usize,
        /// `align_of::<(K, V)>()` = `max(align K, align V)` — slab allocation align.
        pair_align: usize,
        /// Byte offset of the value within a `(K, V)` slot (facet
        /// `value_offset_in_pair`). The key is always at offset 0.
        value_offset: usize,
        /// IR block decoding one pair into the current slab slot
        /// (key at offset 0, value at `value_offset`).
        body_block: usize,
    },

    // -----------------------------------------------------------------------
    // Slow path
    // -----------------------------------------------------------------------

    // r[impl schema.exchange.required]
    /// Fall back to the reflective interpreter for a shape that the IR
    /// cannot lower. The interpreter recognises this instruction and
    /// invokes the reflective path with the embedded plan.
    SlowPath {
        shape: &'static Shape,
        plan: Box<TranslationPlan>,
        dst_offset: usize,
    },

    // -----------------------------------------------------------------------
    // Control flow
    // -----------------------------------------------------------------------
    /// Unconditional jump to `block_id`.
    Jump { block_id: usize },

    /// End of the current block — execution falls through to the next
    /// instruction in the parent context (return from block call).
    Return,

    /// Recursively decode `top_shape` (the program's root) into the
    /// destination slot at `dst_offset`. Emitted by the lowerer when it
    /// detects direct self-recursion (`Box<Self>`, `Vec<Self>`, etc.) so
    /// the IR doesn't infinitely inline the same shape.
    ///
    /// Interpreter: re-runs block 0 of the same program with `out_ptr`
    /// adjusted by `dst_offset`. JIT: emits a self-recursive call to the
    /// same function being compiled.
    CallSelf { dst_offset: usize },

    /// Tail-position equivalent of `CallSelf`: the recursive call is the
    /// last work the function does, so we can reuse the current stack
    /// frame instead of pushing a new one. Both interpreter and JIT
    /// implement it as "update `out_ptr` to the new slot and jump back
    /// to block 0" rather than a real call/return pair.
    ///
    /// Lowering only emits this when it can statically prove tail
    /// position (e.g. the last `Box<Self>` field of an enum variant
    /// whose enclosing block has nothing after it but `Return`). For
    /// branching recursion like `enum Tree { Node(Box<Self>, Box<Self>) }`
    /// only the trailing `Box<Self>` becomes `TailCallSelf`; the leading
    /// one stays as a regular `CallSelf`.
    TailCallSelf { dst_offset: usize },
}

// ---------------------------------------------------------------------------
// Basic block
// ---------------------------------------------------------------------------

/// A linear sequence of `DecodeOp` instructions.
#[derive(Debug, Clone, Default)]
pub struct DecodeBlock {
    pub ops: Vec<DecodeOp>,
}

// ---------------------------------------------------------------------------
// Program
// ---------------------------------------------------------------------------

/// A fully-lowered decode program for one root type.
///
/// Block 0 is always the entry point.
#[derive(Debug, Clone)]
pub struct DecodeProgram {
    pub blocks: Vec<DecodeBlock>,
    /// Total size in bytes of the root destination struct.
    pub root_size: usize,
    /// Required alignment of the root destination.
    pub root_align: usize,
    /// Root shape this program decodes. `CallSelf` ops re-enter block 0,
    /// so the JIT and interpreter can dispatch self-recursive calls
    /// back into this same program.
    pub top_shape: Option<&'static Shape>,
    /// Shapes whose `lower_value` is currently on the lowering stack —
    /// transient state used to detect direct self-recursion (`Box<Self>`,
    /// `Vec<Self>`) and emit a `CallSelf` op instead of inlining the same
    /// shape forever. Empty after lowering finishes.
    #[doc(hidden)]
    pub lowering_in_progress: HashSet<&'static Shape>,
}

impl DecodeProgram {
    fn new_block(&mut self) -> usize {
        let id = self.blocks.len();
        self.blocks.push(DecodeBlock::default());
        id
    }

    fn emit(&mut self, block: usize, op: DecodeOp) {
        self.blocks[block].ops.push(op);
    }
}

// ---------------------------------------------------------------------------
// Lowering: TranslationPlan + Shape → DecodeProgram          (Task #2)
// ---------------------------------------------------------------------------

/// Error returned by the lowering pass.
#[derive(Debug)]
pub enum LowerError {
    /// The shape does not have a known sized layout.
    UnsizedShape,
    /// The enum representation is not stable (Rust or NPO repr).
    UnstableEnumRepr,
    /// A required schema lookup failed during skip-op construction.
    SchemaMissing,
    /// The shape is structurally lowerable but a required calibrated
    /// descriptor or per-shape support is missing. Carries a human-readable
    /// reason. Callers that want a pure-JIT pipeline should treat this as
    /// fatal so the missing calibration gets added rather than silently
    /// degrading to a slow path.
    Unsupported(String),
}

// r[impl schema.errors.early-detection]
/// Lower a validated `TranslationPlan` and corresponding local `Shape` into a
/// `DecodeProgram`.
///
/// `registry` is the *remote* schema registry, used only to resolve skip-op
/// schema kinds for fields that exist on the remote side but not locally.
///
/// The plan must already be validated (no compatibility errors). This pass
/// does NOT re-check structural compatibility.
pub fn lower(
    plan: &TranslationPlan,
    shape: &'static Shape,
    registry: &SchemaRegistry,
) -> Result<DecodeProgram, LowerError> {
    lower_with_cal(plan, shape, registry, None, BorrowMode::Owned)
}

/// Like `lower` but with an optional calibration registry.
///
/// When `cal` is provided, `Vec<T>` types whose element shape has been
/// pre-registered in the registry (via `register_for_shape`) will be lowered
/// to `ReadListLen` + `AllocBacking` + element loop + `CommitListLen` instead
/// of `SlowPath`.
pub fn lower_with_cal(
    plan: &TranslationPlan,
    shape: &'static Shape,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
) -> Result<DecodeProgram, LowerError> {
    let layout = shape
        .layout
        .sized_layout()
        .map_err(|_| LowerError::UnsizedShape)?;
    let mut program = DecodeProgram {
        blocks: vec![DecodeBlock::default()],
        root_size: layout.size(),
        root_align: layout.align(),
        top_shape: Some(shape),
        lowering_in_progress: HashSet::new(),
    };

    let entry = 0;
    lower_value(
        plan,
        shape,
        registry,
        cal,
        borrow_mode,
        &mut program,
        entry,
        0,
    )?;
    program.emit(entry, DecodeOp::Return);
    debug_assert!(program.lowering_in_progress.is_empty());

    mark_tail_calls(&mut program);

    Ok(program)
}

/// Post-lowering pass: rewrite `CallSelf` ops that are statically in tail
/// position to `TailCallSelf`. The JIT lowers `TailCallSelf` to
/// `out_ptr = new_dst; jump body_entry`, reusing the current frame instead
/// of pushing a new one.
///
/// A block is "tail-context" if reaching its last non-Return op means the
/// only remaining work is to return up the stack. Block 0 is tail-context
/// by definition. From any tail-context block, the last non-Return op's
/// child blocks (variant blocks of `BranchOnVariant`, body of `AllocBoxed`,
/// empty block of `ReadListLen`, ...) are also tail-context. List/array
/// loop bodies are never tail-context.
///
/// A `CallSelf` is rewritten to `TailCallSelf` iff it's the last non-Return
/// op of a tail-context block. For `enum Tree { Node(Box<Self>, Box<Self>) }`
/// only the trailing `Box<Self>` recursion gets the tail rewrite — the
/// leading one stays a regular call so its caller can continue with the
/// second `AllocBoxed`.
fn mark_tail_calls(program: &mut DecodeProgram) {
    let mut tail_blocks: HashSet<usize> = HashSet::new();
    tail_blocks.insert(0);

    let mut changed = true;
    while changed {
        changed = false;
        for block_idx in 0..program.blocks.len() {
            if !tail_blocks.contains(&block_idx) {
                continue;
            }
            // Find the last non-Return op.
            let last_real = program.blocks[block_idx]
                .ops
                .iter()
                .rposition(|op| !matches!(op, DecodeOp::Return));
            let Some(pos) = last_real else { continue };
            let op = program.blocks[block_idx].ops[pos].clone();
            match op {
                DecodeOp::AllocBoxed { body_block, .. } => {
                    changed |= tail_blocks.insert(body_block);
                }
                DecodeOp::BranchOnVariant { variant_blocks, .. } => {
                    for (_, vb) in variant_blocks.iter() {
                        if *vb == usize::MAX {
                            continue;
                        }
                        if tail_blocks.insert(*vb) {
                            changed = true;
                        }
                    }
                }
                DecodeOp::ReadListLen { empty_block, .. } => {
                    changed |= tail_blocks.insert(empty_block);
                }
                DecodeOp::DecodeOption { some_block, .. } => {
                    changed |= tail_blocks.insert(some_block);
                }
                DecodeOp::DecodeResult {
                    ok_block,
                    err_block,
                    ..
                }
                | DecodeOp::DecodeResultInit {
                    ok_block,
                    err_block,
                    ..
                } => {
                    if tail_blocks.insert(ok_block) {
                        changed = true;
                    }
                    if tail_blocks.insert(err_block) {
                        changed = true;
                    }
                }
                DecodeOp::Jump { block_id } => {
                    changed |= tail_blocks.insert(block_id);
                }
                _ => {}
            }
        }
    }

    for &block_idx in &tail_blocks {
        let ops = &mut program.blocks[block_idx].ops;
        let Some(pos) = ops.iter().rposition(|op| !matches!(op, DecodeOp::Return)) else {
            continue;
        };
        if let DecodeOp::CallSelf { dst_offset } = ops[pos] {
            ops[pos] = DecodeOp::TailCallSelf { dst_offset };
        }
    }
}

#[allow(clippy::too_many_arguments)]
fn lower_value(
    plan: &TranslationPlan,
    shape: &'static Shape,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    // Cycle detection: if `shape` is already being lowered higher up the
    // stack, we're staring at recursion (`Box<Self>`, `Vec<Self>`, ...).
    // For self-recursion to the program's root we emit `CallSelf` so the
    // JIT/interpreter recurses through the compiled function/program. For
    // mutual recursion (in-progress but not the root) we punt to the
    // reflective slow path — supporting that needs a per-shape compile
    // pipeline we don't have on the decode side.
    if program.lowering_in_progress.contains(&shape) {
        if program.top_shape == Some(shape) {
            trace_cycle_emission(
                "decode",
                "CallSelf",
                shape,
                program.top_shape,
                &program.lowering_in_progress,
            );
            program.emit(block, DecodeOp::CallSelf { dst_offset });
        } else {
            trace_cycle_emission(
                "decode",
                "SlowPath",
                shape,
                program.top_shape,
                &program.lowering_in_progress,
            );
            program.emit(
                block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset,
                },
            );
        }
        return Ok(());
    }
    program.lowering_in_progress.insert(shape);
    let result = lower_value_inner(
        plan,
        shape,
        registry,
        cal,
        borrow_mode,
        program,
        block,
        dst_offset,
    );
    program.lowering_in_progress.remove(&shape);
    result
}

#[allow(clippy::too_many_arguments)]
fn lower_value_inner(
    plan: &TranslationPlan,
    shape: &'static Shape,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    // `Result<T, E>` is exposed as an opaque/proxy-like user shape by Facet,
    // but its postcard ABI is structural. Route it by Def before generic
    // proxy/opaque handling.
    if let facet_core::Def::Result(_) = shape.def {
        return lower_def(
            plan,
            shape,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        );
    }

    if shape.opaque_adapter.is_some() {
        program.emit(block, DecodeOp::ReadOpaque { shape, dst_offset });
        return Ok(());
    }

    if shape.proxy.is_some() {
        program.emit(
            block,
            DecodeOp::SlowPath {
                shape,
                plan: Box::new(clone_plan(plan)),
                dst_offset,
            },
        );
        return Ok(());
    }

    // Transparent wrappers — pass through to inner shape
    if shape.is_transparent() {
        if let Type::User(UserType::Struct(st)) = shape.ty
            && let Some(inner_field) = st.fields.first()
        {
            let inner_shape = inner_field.shape();
            return lower_value(
                plan,
                inner_shape,
                registry,
                cal,
                borrow_mode,
                program,
                block,
                dst_offset,
            );
        }
        // Transparent wrapper with no first field (e.g. dynamically generated
        // transparent struct with no inner slot) — SlowPath by design.
        program.emit(
            block,
            DecodeOp::SlowPath {
                shape,
                plan: Box::new(clone_plan(plan)),
                dst_offset,
            },
        );
        return Ok(());
    }

    // Scalars
    if let Some(scalar) = shape.scalar_type() {
        match scalar {
            // `std::string::String` — 3-slot (ptr/len/cap) heap container.
            // Uses the calibrated String descriptor; layout is whatever
            // calibration measured for this build's `String` repr.
            facet_core::ScalarType::String if shape.is_type::<String>() => {
                let cal = cal.ok_or_else(|| {
                    LowerError::Unsupported(
                        "ScalarType::String requires a CalibrationRegistry".into(),
                    )
                })?;
                let handle = cal.string_descriptor_handle().ok_or_else(|| {
                    LowerError::Unsupported(
                        "String descriptor not registered in CalibrationRegistry".into(),
                    )
                })?;
                program.emit(
                    block,
                    DecodeOp::ReadString {
                        dst_offset,
                        descriptor: OpaqueDescriptorId(handle.0),
                    },
                );
                return Ok(());
            }
            // `Cow<str>` — handled by its own helper which knows the layout
            // and picks borrowed vs owned at runtime.
            facet_core::ScalarType::CowStr => {
                program.emit(
                    block,
                    DecodeOp::ReadCowStr {
                        dst_offset,
                        borrowed: borrow_mode == BorrowMode::Borrowed,
                    },
                );
                return Ok(());
            }
            // `&str` — 2-slot fat pointer (ptr, len). Layout is fixed by
            // Rust's wide-pointer ABI, so no calibration is needed. Always
            // points into the input buffer; the caller (e.g. SelfRef in
            // `deserialize_postcard`) is responsible for keeping it alive
            // for the lifetime of the decoded value.
            facet_core::ScalarType::Str => {
                program.emit(block, DecodeOp::ReadStrRef { dst_offset });
                return Ok(());
            }
            _ => {}
        }

        if let Some(prim) = wire_primitive_from_scalar(scalar) {
            // `WirePrimitive::String` is reachable here only via custom
            // `ScalarType::String` types that aren't `std::string::String`
            // (handled above) — those would need a per-shape calibration.
            // Reject loudly rather than silently writing the std-String
            // layout into a destination of unknown size.
            if matches!(prim, WirePrimitive::String) {
                return Err(LowerError::Unsupported(format!(
                    "ScalarType::String on non-`std::string::String` type {shape} requires \
                     a per-shape string descriptor; not yet calibrated"
                )));
            }
            program.emit(block, DecodeOp::ReadScalar { prim, dst_offset });
            return Ok(());
        }
        // Scalar kind not representable as a postcard primitive
        // (e.g. SocketAddr, IpAddr, ConstTypeId) — SlowPath by design.
        // These types have no canonical postcard encoding; the reflective
        // interpreter handles them via their own vtable deserialize paths.
        program.emit(
            block,
            DecodeOp::SlowPath {
                shape,
                plan: Box::new(clone_plan(plan)),
                dst_offset,
            },
        );
        return Ok(());
    }

    match shape.def {
        facet_core::Def::Option(_)
        | facet_core::Def::Array(_)
        | facet_core::Def::List(_)
        | facet_core::Def::Pointer(_) => {
            return lower_def(
                plan,
                shape,
                registry,
                cal,
                borrow_mode,
                program,
                block,
                dst_offset,
            );
        }
        _ => {}
    }

    // User types
    match shape.ty {
        Type::User(UserType::Struct(st)) => lower_struct(
            plan,
            st,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        ),
        Type::User(UserType::Enum(et)) => lower_enum(
            plan,
            shape,
            et,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        ),
        _ => lower_def(
            plan,
            shape,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        ),
    }
}

#[allow(clippy::too_many_arguments)]
fn lower_def(
    plan: &TranslationPlan,
    shape: &'static Shape,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    use facet_core::Def;

    match shape.def {
        Def::Option(opt_def) => lower_option(
            plan,
            shape,
            opt_def,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        ),
        Def::Array(arr_def) => lower_array(
            plan,
            arr_def,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        ),
        Def::List(list_def) => lower_list(
            plan,
            shape,
            list_def,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        ),
        Def::Pointer(ptr_def) => lower_pointer(
            plan,
            shape,
            ptr_def,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        ),
        Def::Result(result_def) => lower_result(
            plan,
            shape,
            result_def,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        ),
        Def::Map(map_def) => lower_map(
            plan,
            shape,
            map_def,
            registry,
            cal,
            borrow_mode,
            program,
            block,
            dst_offset,
        ),
        _ => {
            // Def::Set, Def::Slice, Def::NdArray, Def::DynamicValue,
            // Def::Undefined — SlowPath.
            //
            // Set: same slab strategy as Map would apply (SetVTable exposes
            // from_slice); a native lowering is a fast-follow once DecodeMap
            // lands. Slice: borrowed &[T] — a separate piece of work.
            // NdArray/DynamicValue/Undefined: no postcard ABI defined.
            program.emit(
                block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset,
                },
            );
            Ok(())
        }
    }
}

/// Lower a map (`BTreeMap`/`HashMap`) to a `DecodeMap` op plus a body block.
///
/// The body block decodes one `(K, V)` pair into a scratch-slab slot. At
/// runtime `DecodeMap` reads the entry count, decodes every pair natively into
/// the slab, then calls facet's `from_pair_slice` to build the map in one shot.
///
/// Falls back to `SlowPath` when the map type does not expose a
/// `from_pair_slice` constructor (no batch builder ⇒ no slab strategy).
#[allow(clippy::too_many_arguments)]
fn lower_map(
    plan: &TranslationPlan,
    shape: &'static Shape,
    map_def: facet_core::MapDef,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    let slow_path = |program: &mut DecodeProgram| {
        program.emit(
            block,
            DecodeOp::SlowPath {
                shape,
                plan: Box::new(clone_plan(plan)),
                dst_offset,
            },
        );
    };

    // The slab strategy needs facet's batch constructor. Without it (a map
    // type that only exposes per-entry `insert`) fall back to the interpreter.
    let Some(from_pair_slice) = map_def.vtable.from_pair_slice else {
        slow_path(program);
        return Ok(());
    };

    // Key and value must be sized — the slab holds inline `(K, V)` slots.
    let (Ok(k_layout), Ok(v_layout)) = (
        map_def.k().layout.sized_layout(),
        map_def.v().layout.sized_layout(),
    ) else {
        slow_path(program);
        return Ok(());
    };

    let pair_stride = map_def.vtable.pair_stride;
    let value_offset = map_def.vtable.value_offset_in_pair;
    // Alignment of `(K, V)` is the max of the field alignments (true for both
    // repr(Rust) and repr(C) aggregates). `.max(1)` guards the all-ZST case.
    let pair_align = k_layout.align().max(v_layout.align()).max(1);

    // A degenerate `pair_stride` of 0 (both K and V are ZSTs) means the slab
    // carries no bytes; `from_pair_slice` still works (it reads `count` ZST
    // pairs). The codegen/interpreter treat stride 0 as "no allocation".

    // Key/value translation plans, if the schemas differ on either side.
    let (key_plan, value_plan): (&TranslationPlan, &TranslationPlan) = match plan {
        TranslationPlan::Map { key, value } => (key, value),
        _ => (&TranslationPlan::Identity, &TranslationPlan::Identity),
    };

    let body_block = program.new_block();
    program.emit(
        block,
        DecodeOp::DecodeMap {
            dst_offset,
            from_pair_slice,
            pair_stride,
            pair_align,
            value_offset,
            body_block,
        },
    );

    // Body: decode the key at slot offset 0, the value at `value_offset`.
    lower_value(
        key_plan,
        map_def.k(),
        registry,
        cal,
        borrow_mode,
        program,
        body_block,
        0,
    )?;
    lower_value(
        value_plan,
        map_def.v(),
        registry,
        cal,
        borrow_mode,
        program,
        body_block,
        value_offset,
    )?;
    program.emit(body_block, DecodeOp::Return);
    Ok(())
}

#[allow(clippy::too_many_arguments)]
fn lower_result(
    plan: &TranslationPlan,
    shape: &'static Shape,
    result_def: facet_core::ResultDef,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    let (ok_plan, err_plan) = match plan {
        TranslationPlan::Enum { nested, .. } => (
            nested.get(&0).unwrap_or(&TranslationPlan::Identity),
            nested.get(&1).unwrap_or(&TranslationPlan::Identity),
        ),
        TranslationPlan::Identity => (&TranslationPlan::Identity, &TranslationPlan::Identity),
        _ => {
            program.emit(
                block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset,
                },
            );
            return Ok(());
        }
    };

    let ok_block = program.new_block();
    let err_block = program.new_block();

    if let Some(layout) = calibrate_result_layout(shape, result_def) {
        program.emit(
            block,
            DecodeOp::DecodeResult {
                dst_offset,
                ok_block,
                err_block,
                ok_offset: layout.ok_offset,
                err_offset: layout.err_offset,
                ok_bytes: layout.ok_bytes,
                err_bytes: layout.err_bytes,
            },
        );
    } else {
        let Ok(ok_layout) = result_def.t.layout.sized_layout() else {
            program.emit(
                block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset,
                },
            );
            return Ok(());
        };
        let Ok(err_layout) = result_def.e.layout.sized_layout() else {
            program.emit(
                block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset,
                },
            );
            return Ok(());
        };

        program.emit(
            block,
            DecodeOp::DecodeResultInit {
                dst_offset,
                ok_block,
                err_block,
                ok_size: ok_layout.size(),
                ok_align: ok_layout.align(),
                err_size: err_layout.size(),
                err_align: err_layout.align(),
                init_ok_fn: result_def.vtable.init_ok,
                init_err_fn: result_def.vtable.init_err,
            },
        );
    }

    lower_value(
        ok_plan,
        result_def.t,
        registry,
        cal,
        borrow_mode,
        program,
        ok_block,
        0,
    )?;
    program.emit(ok_block, DecodeOp::Return);

    lower_value(
        err_plan,
        result_def.e,
        registry,
        cal,
        borrow_mode,
        program,
        err_block,
        0,
    )?;
    program.emit(err_block, DecodeOp::Return);

    Ok(())
}

// r[impl schema.translation.reorder]
// r[impl schema.translation.skip-unknown]
#[allow(clippy::too_many_arguments)]
fn lower_struct(
    plan: &TranslationPlan,
    st: facet_core::StructType,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    let (field_ops, nested) = match plan {
        TranslationPlan::Struct { field_ops, nested }
        | TranslationPlan::Tuple { field_ops, nested } => (field_ops.as_slice(), nested),
        TranslationPlan::Identity => {
            let identity = build_identity_plan_for_struct(st);
            return lower_struct(
                &identity,
                st,
                registry,
                cal,
                borrow_mode,
                program,
                block,
                dst_offset,
            );
        }
        _ => {
            // A validated plan for a struct is always Struct | Tuple | Identity.
            // Any other variant indicates a bug in the caller.
            return Err(LowerError::SchemaMissing);
        }
    };

    let mut matched = vec![false; st.fields.len()];
    for op in field_ops {
        match op {
            FieldOp::Read { local_index } => {
                matched[*local_index] = true;
                let field = &st.fields[*local_index];
                let field_shape = field.shape();
                let field_offset = dst_offset + field.offset;
                let sub_plan = nested
                    .get(local_index)
                    .unwrap_or(&TranslationPlan::Identity);
                lower_value(
                    sub_plan,
                    field_shape,
                    registry,
                    cal,
                    borrow_mode,
                    program,
                    block,
                    field_offset,
                )?;
            }
            FieldOp::Skip { type_ref } => {
                let kind = type_ref
                    .resolve_kind(registry)
                    .ok_or(LowerError::SchemaMissing)?;
                program.emit(block, DecodeOp::SkipValue { kind });
            }
        }
    }

    // r[impl schema.translation.fill-defaults]
    // Local fields with no corresponding remote field need a Default-fill:
    // plan-build has already verified they're not required (i.e. have a
    // `#[facet(default)]` attribute), so `call_default_in_place` will succeed.
    for (i, field) in st.fields.iter().enumerate() {
        if !matched[i] {
            program.emit(
                block,
                DecodeOp::WriteDefault {
                    shape: field.shape(),
                    dst_offset: dst_offset + field.offset,
                },
            );
        }
    }

    Ok(())
}

// r[impl schema.translation.enum]
// r[impl schema.translation.enum.unknown-variant]
// r[impl schema.translation.enum.payload-compat]
#[allow(clippy::too_many_arguments)]
fn lower_enum(
    plan: &TranslationPlan,
    shape: &'static Shape,
    et: facet_core::EnumType,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    // EnumRepr::Rust and RustNPO have compiler-chosen discriminant layout; we
    // cannot emit a BranchOnVariant op. Emit SlowPath for the whole enum value
    // (e.g. Option<T>, which has RustNPO/Rust repr) so the field decodes via the
    // reflective interpreter without aborting the entire stub compilation.
    let Some(tag_width) = tag_width_from_enum_repr(et.enum_repr) else {
        program.emit(
            block,
            DecodeOp::SlowPath {
                shape,
                plan: Box::new(clone_plan(plan)),
                dst_offset,
            },
        );
        return Ok(());
    };

    let (variant_map, variant_plans, nested) = match plan {
        TranslationPlan::Enum {
            variant_map,
            variant_plans,
            nested,
        } => (variant_map, variant_plans, nested),
        TranslationPlan::Identity => {
            let identity = crate::build_identity_plan(shape);
            return lower_enum(
                &identity,
                shape,
                et,
                registry,
                cal,
                borrow_mode,
                program,
                block,
                dst_offset,
            );
        }
        _ => {
            // Non-Enum plan for an enum type — SlowPath by design.
            // Only `TranslationPlan::Enum { .. }` carries variant_map and
            // variant_plans, which are required to emit `BranchOnVariant`.
            // Any other plan variant (e.g. a bare Identity) signals that the
            // plan was built without enum awareness and must use the reflective
            // interpreter.
            program.emit(
                block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset,
                },
            );
            return Ok(());
        }
    };

    // Emit discriminant read
    program.emit(block, DecodeOp::ReadDiscriminant);

    // Build per-variant blocks
    let mut variant_blocks: Vec<(u64, usize)> = Vec::new();
    for (remote_idx, maybe_local) in variant_map.iter().enumerate() {
        let Some(local_idx) = maybe_local else {
            // Unknown remote variant → push sentinel (interpreter errors at runtime)
            variant_blocks.push((u64::MAX, usize::MAX));
            continue;
        };
        let local_variant = &et.variants[*local_idx];

        // Discriminant value for the local variant
        let local_disc = local_variant
            .discriminant
            .map(|d| d as u64)
            .unwrap_or(*local_idx as u64);

        let variant_block = program.new_block();
        variant_blocks.push((local_disc, variant_block));

        // Lower variant fields into the variant block
        if let Some(variant_plan) = variant_plans.get(&remote_idx) {
            lower_struct(
                variant_plan,
                local_variant.data,
                registry,
                cal,
                borrow_mode,
                program,
                variant_block,
                dst_offset,
            )?;
        } else if let Some(inner_plan) = nested.get(local_idx) {
            // Newtype variant — single field
            if let Some(field) = local_variant.data.fields.first() {
                let field_offset = dst_offset + field.offset;
                lower_value(
                    inner_plan,
                    field.shape(),
                    registry,
                    cal,
                    borrow_mode,
                    program,
                    variant_block,
                    field_offset,
                )?;
            }
        } else {
            // Identity: read fields in order
            let identity = build_identity_plan_for_struct(local_variant.data);
            lower_struct(
                &identity,
                local_variant.data,
                registry,
                cal,
                borrow_mode,
                program,
                variant_block,
                dst_offset,
            )?;
        }

        program.emit(variant_block, DecodeOp::Return);
    }

    program.emit(
        block,
        DecodeOp::BranchOnVariant {
            tag_offset: dst_offset,
            tag_width,
            variant_table: variant_map.clone(),
            variant_blocks,
        },
    );

    Ok(())
}

struct ScratchBuf {
    ptr: *mut u8,
    layout: std::alloc::Layout,
}

impl ScratchBuf {
    #[allow(unsafe_code)]
    fn new(layout: std::alloc::Layout) -> Option<Self> {
        let ptr = unsafe { std::alloc::alloc_zeroed(layout) };
        if ptr.is_null() {
            None
        } else {
            Some(Self { ptr, layout })
        }
    }

    #[allow(unsafe_code)]
    fn new_filled(layout: std::alloc::Layout, fill: u8) -> Option<Self> {
        let ptr = unsafe { std::alloc::alloc(layout) };
        if ptr.is_null() {
            return None;
        }
        unsafe { std::ptr::write_bytes(ptr, fill, layout.size()) };
        Some(Self { ptr, layout })
    }

    #[allow(unsafe_code)]
    fn to_bytes(&self) -> Box<[u8]> {
        unsafe { std::slice::from_raw_parts(self.ptr as *const u8, self.layout.size()) }
            .to_vec()
            .into_boxed_slice()
    }
}

impl Drop for ScratchBuf {
    #[allow(unsafe_code)]
    fn drop(&mut self) {
        unsafe { std::alloc::dealloc(self.ptr, self.layout) };
    }
}

struct CalibratedOptionLayout {
    inner_offset: usize,
    none_bytes: Box<[u8]>,
    some_bytes: Box<[u8]>,
    /// Positions (and expected None values) of the discriminator bytes — bytes
    /// rustc reliably writes in *both* `init_none` and `init_some`, whose values
    /// differ between the two variants. Used by the JIT encode fast path to
    /// classify an `Option<T>` at runtime without calling `is_some_fn`.
    ///
    /// Computed by probing each init fn twice (once over zero-filled memory,
    /// once over `0xFF`-filled memory): bytes whose value depends on the fill
    /// are padding / uninitialized and are excluded from the discriminator.
    tag_bytes: Box<[(usize, u8)]>,
}

struct CalibratedResultLayout {
    ok_offset: usize,
    err_offset: usize,
    ok_bytes: Box<[u8]>,
    err_bytes: Box<[u8]>,
}

#[allow(unsafe_code)]
fn calibrate_option_layout(
    shape: &'static Shape,
    opt_def: facet_core::OptionDef,
) -> Option<CalibratedOptionLayout> {
    let opt_layout = shape.layout.sized_layout().ok()?;
    let inner_layout = opt_def.t.layout.sized_layout().ok()?;

    // Probe each init fn over several distinct outer fills. A byte is
    // "consistently written" only if it has the same value across every probe
    // of that variant — padding and uninit bytes that `ptr::write` doesn't
    // touch follow the fill and get filtered out. More than two fills helps
    // with cases where `ptr::write` copies undef bytes whose value happens
    // to match across two runs.
    const FILLS: [u8; 4] = [0x00, 0xFF, 0x5A, 0xA5];

    let mut none_probes: Vec<Box<[u8]>> = Vec::with_capacity(FILLS.len());
    for &fill in &FILLS {
        none_probes.push(probe_option_none(shape, opt_def, opt_layout, fill)?.0);
    }

    // Niche-optimized fast path. When `size_of::<Option<T>>() == size_of::<T>()`,
    // the layout has no separate discriminator slot — Rust packs the tag into a
    // forbidden bit pattern of T (e.g. null-pointer niche of `&T`/`&str`/`Box<T>`
    // /`Vec<T>`/`String`, or zero-niche of `NonZero*`). This means:
    //   1. T sits at offset 0 inside the option.
    //   2. `init_none` writes bytes that include the niche value (typically 0).
    //   3. When the inner decode runs into the same slot for Some, it overwrites
    //      whatever bytes the niche occupies with non-niche values.
    //
    // The standard probe-Some path requires `T: Default` via facet's vtable to
    // construct a Some value; reference types (`&T`, `&str`, `&[T]`) explicitly
    // set `default_in_place: None` in their vtable, so probe-Some fails for them.
    // For niche-optimized Options we don't actually need to probe Some — we can
    // derive `tag_bytes` from the None probe alone (any byte init_none
    // consistently zeros must be a niche byte, since the niche must contain a
    // forbidden Some pattern by definition).
    if opt_layout.size() == inner_layout.size() {
        let mut tag_bytes: Vec<(usize, u8)> = Vec::new();
        for i in 0..opt_layout.size() {
            let val = none_probes[0][i];
            let consistent = none_probes.iter().all(|p| p[i] == val);
            if consistent && val == 0 {
                tag_bytes.push((i, 0));
            }
        }
        if tag_bytes.is_empty() {
            return None;
        }
        let none_bytes = none_probes.into_iter().next().unwrap();
        // For niche-optimized layouts the inner decode at offset 0 fully
        // overwrites the niche bytes, so `some_bytes` and `none_bytes` can be
        // identical — the JIT writes `some_bytes` first, then the inner decode
        // emits the actual value on top.
        let some_bytes = none_bytes.clone();
        return Some(CalibratedOptionLayout {
            inner_offset: 0,
            none_bytes,
            some_bytes,
            tag_bytes: tag_bytes.into_boxed_slice(),
        });
    }

    let mut some_probes: Vec<Box<[u8]>> = Vec::with_capacity(FILLS.len());
    let mut inner_offset: Option<usize> = None;
    for &fill in &FILLS {
        let (bytes, off) = probe_option_some(shape, opt_def, opt_layout, inner_layout, fill)?;
        match inner_offset {
            Some(prev) if prev != off => return None,
            _ => inner_offset = Some(off),
        }
        some_probes.push(bytes);
    }

    let inner_offset = inner_offset?;
    if inner_offset.checked_add(inner_layout.size())? > opt_layout.size() {
        return None;
    }

    let mut candidates: Vec<(usize, u8, u8)> = Vec::new();
    for i in 0..opt_layout.size() {
        let none_val = none_probes[0][i];
        let some_val = some_probes[0][i];
        let none_written = none_probes.iter().all(|p| p[i] == none_val);
        let some_written = some_probes.iter().all(|p| p[i] == some_val);
        if none_written && some_written && none_val != some_val {
            candidates.push((i, none_val, some_val));
        }
    }

    // Prune spurious candidates with `is_some_fn` as the oracle: for each
    // candidate offset, start from a fresh None buffer and flip only that
    // byte to its calibrated Some value. If `is_some_fn` still reports
    // None, that byte isn't a real discriminator — some other part of the
    // representation (padding or probe-stable undef) just happened to
    // differ consistently across our probes.
    let mut tag_bytes: Vec<(usize, u8)> = Vec::new();
    for &(i, none_val, some_val) in &candidates {
        let oracle_buf = ScratchBuf::new_filled(opt_layout, 0x00)?;
        unsafe {
            (opt_def.vtable.init_none)(facet_core::PtrUninit::new(oracle_buf.ptr as *mut ()))
        };
        unsafe { oracle_buf.ptr.add(i).write(some_val) };
        let is_some = unsafe {
            (opt_def.vtable.is_some)(facet_core::PtrConst::new(oracle_buf.ptr as *const ()))
        };
        unsafe { shape.call_drop_in_place(facet_core::PtrMut::new(oracle_buf.ptr as *mut ())) };
        if is_some {
            tag_bytes.push((i, none_val));
        }
    }

    if tag_bytes.is_empty() {
        return None;
    }

    Some(CalibratedOptionLayout {
        inner_offset,
        none_bytes: none_probes.into_iter().next().unwrap(),
        some_bytes: some_probes.into_iter().next().unwrap(),
        tag_bytes: tag_bytes.into_boxed_slice(),
    })
}

/// Probe a `None`-initialized Option over memory pre-filled with `fill`.
///
/// Returns `(option_bytes, ())`; the second tuple slot exists only to mirror
/// `probe_option_some` for call-site symmetry.
#[allow(unsafe_code)]
fn probe_option_none(
    shape: &'static Shape,
    opt_def: facet_core::OptionDef,
    opt_layout: std::alloc::Layout,
    fill: u8,
) -> Option<(Box<[u8]>, ())> {
    let buf = ScratchBuf::new_filled(opt_layout, fill)?;
    unsafe { (opt_def.vtable.init_none)(facet_core::PtrUninit::new(buf.ptr as *mut ())) };
    let bytes = buf.to_bytes();
    unsafe { shape.call_drop_in_place(facet_core::PtrMut::new(buf.ptr as *mut ())) };
    Some((bytes, ()))
}

/// Probe a `Some(T::default())`-initialized Option over memory pre-filled with
/// `fill`. Returns `(option_bytes, inner_offset)`.
#[allow(unsafe_code)]
fn probe_option_some(
    shape: &'static Shape,
    opt_def: facet_core::OptionDef,
    opt_layout: std::alloc::Layout,
    inner_layout: std::alloc::Layout,
    fill: u8,
) -> Option<(Box<[u8]>, usize)> {
    let buf = ScratchBuf::new_filled(opt_layout, fill)?;
    let inner_buf = ScratchBuf::new(inner_layout)?;
    unsafe {
        opt_def
            .t
            .call_default_in_place(facet_core::PtrUninit::new(inner_buf.ptr as *mut ()))?
    };
    unsafe {
        (opt_def.vtable.init_some)(
            facet_core::PtrUninit::new(buf.ptr as *mut ()),
            facet_core::PtrMut::new(inner_buf.ptr as *mut ()),
        )
    };

    let ret_ptr =
        unsafe { (opt_def.vtable.get_value)(facet_core::PtrConst::new(buf.ptr as *const ())) };
    let base_ptr = buf.ptr as *const u8;
    let end_ptr = unsafe { base_ptr.add(opt_layout.size()) };
    if ret_ptr < base_ptr || ret_ptr > end_ptr {
        return None;
    }
    let inner_offset = ret_ptr as usize - base_ptr as usize;

    let bytes = buf.to_bytes();
    unsafe { shape.call_drop_in_place(facet_core::PtrMut::new(buf.ptr as *mut ())) };
    Some((bytes, inner_offset))
}

#[allow(unsafe_code)]
fn calibrate_result_layout(
    shape: &'static Shape,
    result_def: facet_core::ResultDef,
) -> Option<CalibratedResultLayout> {
    let result_layout = shape.layout.sized_layout().ok()?;
    let ok_layout = result_def.t.layout.sized_layout().ok()?;
    let err_layout = result_def.e.layout.sized_layout().ok()?;

    let ok_buf = ScratchBuf::new(result_layout)?;
    let ok_inner = ScratchBuf::new(ok_layout)?;
    unsafe {
        result_def
            .t
            .call_default_in_place(facet_core::PtrUninit::new(ok_inner.ptr as *mut ()))?
    };
    unsafe {
        (result_def.vtable.init_ok)(
            facet_core::PtrUninit::new(ok_buf.ptr as *mut ()),
            facet_core::PtrMut::new(ok_inner.ptr as *mut ()),
        )
    };

    let err_buf = ScratchBuf::new(result_layout)?;
    let err_inner = ScratchBuf::new(err_layout)?;
    unsafe {
        result_def
            .e
            .call_default_in_place(facet_core::PtrUninit::new(err_inner.ptr as *mut ()))?
    };
    unsafe {
        (result_def.vtable.init_err)(
            facet_core::PtrUninit::new(err_buf.ptr as *mut ()),
            facet_core::PtrMut::new(err_inner.ptr as *mut ()),
        )
    };

    let base_ptr = ok_buf.ptr as *const u8;
    let end_ptr = unsafe { base_ptr.add(result_layout.size()) };
    let ok_ptr =
        unsafe { (result_def.vtable.get_ok)(facet_core::PtrConst::new(ok_buf.ptr as *const ())) };
    if ok_ptr < base_ptr || ok_ptr > end_ptr {
        return None;
    }
    let ok_offset = ok_ptr as usize - base_ptr as usize;
    if ok_offset.checked_add(ok_layout.size())? > result_layout.size() {
        return None;
    }

    let err_base_ptr = err_buf.ptr as *const u8;
    let err_end_ptr = unsafe { err_base_ptr.add(result_layout.size()) };
    let err_ptr =
        unsafe { (result_def.vtable.get_err)(facet_core::PtrConst::new(err_buf.ptr as *const ())) };
    if err_ptr < err_base_ptr || err_ptr > err_end_ptr {
        return None;
    }
    let err_offset = err_ptr as usize - err_base_ptr as usize;
    if err_offset.checked_add(err_layout.size())? > result_layout.size() {
        return None;
    }

    let ok_bytes = ok_buf.to_bytes();
    let err_bytes = err_buf.to_bytes();
    let _ = unsafe { shape.call_drop_in_place(facet_core::PtrMut::new(ok_buf.ptr as *mut ())) };
    let _ = unsafe { shape.call_drop_in_place(facet_core::PtrMut::new(err_buf.ptr as *mut ())) };

    Some(CalibratedResultLayout {
        ok_offset,
        err_offset,
        ok_bytes,
        err_bytes,
    })
}

#[allow(unsafe_code)]
#[allow(clippy::too_many_arguments)]
fn lower_option(
    plan: &TranslationPlan,
    shape: &'static Shape,
    opt_def: facet_core::OptionDef,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    let inner_plan = match plan {
        TranslationPlan::Option { inner } => inner.as_ref(),
        TranslationPlan::Identity => &TranslationPlan::Identity,
        _ => {
            program.emit(
                block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset,
                },
            );
            return Ok(());
        }
    };

    let Some(layout) = calibrate_option_layout(shape, opt_def) else {
        program.emit(
            block,
            DecodeOp::SlowPath {
                shape,
                plan: Box::new(clone_plan(plan)),
                dst_offset,
            },
        );
        return Ok(());
    };

    let some_block = program.new_block();

    program.emit(
        block,
        DecodeOp::DecodeOption {
            dst_offset,
            inner_offset: layout.inner_offset,
            some_block,
            none_bytes: layout.none_bytes,
            some_bytes: layout.some_bytes,
        },
    );

    // Lower the inner value decode into some_block.
    // The interpreter will call run_block(some_block, inner_ptr) where inner_ptr
    // already points to the inner slot, so dst_offset for inner ops is 0.
    lower_value(
        inner_plan,
        opt_def.t,
        registry,
        cal,
        borrow_mode,
        program,
        some_block,
        0,
    )?;
    program.emit(some_block, DecodeOp::Return);

    Ok(())
}

#[allow(clippy::too_many_arguments)]
fn lower_array(
    plan: &TranslationPlan,
    arr_def: facet_core::ArrayDef,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    let element_plan = match plan {
        TranslationPlan::Array { element } => element.as_ref(),
        TranslationPlan::Identity => &TranslationPlan::Identity,
        _ => &TranslationPlan::Identity,
    };

    let elem_shape = arr_def.t;
    let elem_layout = elem_shape
        .layout
        .sized_layout()
        .map_err(|_| LowerError::UnsizedShape)?;
    let elem_size = elem_layout.size();

    let body_block = program.new_block();

    program.emit(
        block,
        DecodeOp::DecodeArray {
            dst_offset,
            count: arr_def.n,
            elem_size,
            body_block,
        },
    );

    // Lower one element's decode into body_block (base = element pointer, offset = 0).
    lower_value(
        element_plan,
        elem_shape,
        registry,
        cal,
        borrow_mode,
        program,
        body_block,
        0,
    )?;
    program.emit(body_block, DecodeOp::Return);

    Ok(())
}

#[allow(clippy::too_many_arguments)]
fn lower_list(
    plan: &TranslationPlan,
    shape: &'static Shape,
    list_def: facet_core::ListDef,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    let elem_shape = list_def.t;
    let elem_layout = match elem_shape.layout.sized_layout() {
        Ok(l) => l,
        Err(_) => {
            program.emit(
                block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset,
                },
            );
            return Ok(());
        }
    };
    let elem_size = elem_layout.size();

    // Bulk-copy fast path: `Vec<T>` where T's postcard wire format is
    // bit-identical to its in-memory representation. Skips the per-element
    // decode loop entirely and emits a single `memcpy(len * elem_size)` from
    // the input cursor into the freshly-allocated backing.
    //
    // - `u8`/`i8`: 1 byte raw, always eligible.
    // - `f32`/`f64`: postcard-spec'd as fixed little-endian (NOT varint), so
    //   identical to `[f32]` / `[f64]` memory layout on LE hosts only.
    // - `bool`: 1 byte, but must be 0 or 1 — memcpy then bitwise-OR scan to
    //   reject any byte >= 2.
    //
    // Integer types `u16`/`u32`/`u64` and signed counterparts use varint, so
    // they're NOT eligible here — fall through to the per-element path.
    let is_byte_elem = list_def.t.is_type::<u8>() || list_def.t.is_type::<i8>();
    let is_bool_elem = list_def.t.is_type::<bool>();
    let is_float_elem_le = cfg!(target_endian = "little")
        && (list_def.t.is_type::<f32>() || list_def.t.is_type::<f64>());
    if is_byte_elem || is_bool_elem || is_float_elem_le {
        if let Some(cal) = cal
            && let Some(descriptor) = cal.lookup_by_shape(shape)
        {
            let descriptor = OpaqueDescriptorId(descriptor.0);
            program.emit(
                block,
                DecodeOp::ReadFixedVec {
                    dst_offset,
                    descriptor,
                    elem_size,
                    validate_bool: is_bool_elem,
                },
            );
            return Ok(());
        }
        program.emit(
            block,
            DecodeOp::SlowPath {
                shape,
                plan: Box::new(clone_plan(plan)),
                dst_offset,
            },
        );
        return Ok(());
    }

    // Generic Vec<T> — emit calibrated ops when available.
    // lookup_by_shape keys by Shape value (via blanket impl<T: Hash> Hash for &T).
    if let Some(cal) = cal
        && let Some(descriptor_handle) = cal.lookup_by_shape(shape)
    {
        let descriptor = OpaqueDescriptorId(descriptor_handle.0);

        let empty_block = program.new_block();
        let body_block = program.new_block();
        let inner_block = program.new_block();

        program.emit(
            block,
            DecodeOp::ReadListLen {
                descriptor,
                dst_offset,
                empty_block,
                body_block,
            },
        );

        // Empty path: copy calibrated empty bytes.
        program.emit(
            empty_block,
            DecodeOp::MaterializeEmpty {
                dst_offset,
                descriptor,
            },
        );
        program.emit(empty_block, DecodeOp::Return);

        // Non-empty path: allocate backing, then loop body.
        program.emit(
            body_block,
            DecodeOp::AllocBacking {
                dst_offset,
                descriptor,
                body_block: inner_block,
                elem_size,
            },
        );
        program.emit(body_block, DecodeOp::Return);

        // Element body block: decode one element at out_ptr[0], commit len.
        let element_plan = match plan {
            TranslationPlan::List { element } => element.as_ref(),
            _ => &TranslationPlan::Identity,
        };
        lower_value(
            element_plan,
            elem_shape,
            registry,
            Some(cal),
            borrow_mode,
            program,
            inner_block,
            0,
        )?;
        program.emit(
            inner_block,
            DecodeOp::CommitListLen {
                dst_offset,
                descriptor,
            },
        );
        program.emit(inner_block, DecodeOp::Return);

        return Ok(());
    }

    // No calibration — fall back to reflective interpreter.
    program.emit(
        block,
        DecodeOp::SlowPath {
            shape,
            plan: Box::new(clone_plan(plan)),
            dst_offset,
        },
    );
    Ok(())
}

#[allow(clippy::too_many_arguments)]
fn lower_pointer(
    plan: &TranslationPlan,
    shape: &'static Shape,
    ptr_def: facet_core::PointerDef,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
    program: &mut DecodeProgram,
    block: usize,
    dst_offset: usize,
) -> Result<(), LowerError> {
    let pointee_plan = match plan {
        TranslationPlan::Pointer { pointee } => pointee.as_ref(),
        TranslationPlan::Identity => &TranslationPlan::Identity,
        _ => {
            program.emit(
                block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset,
                },
            );
            return Ok(());
        }
    };

    let Some(pointee_shape) = ptr_def.pointee() else {
        // Opaque pointer — slow path.
        program.emit(
            block,
            DecodeOp::SlowPath {
                shape,
                plan: Box::new(clone_plan(plan)),
                dst_offset,
            },
        );
        return Ok(());
    };

    if let facet_core::Def::Slice(slice_def) = pointee_shape.def
        && slice_def.t().is_type::<u8>()
    {
        match ptr_def.known {
            Some(facet_core::KnownPointer::Cow) => {
                program.emit(
                    block,
                    DecodeOp::ReadCowByteSlice {
                        dst_offset,
                        borrowed: borrow_mode == BorrowMode::Borrowed,
                    },
                );
                return Ok(());
            }
            Some(facet_core::KnownPointer::SharedReference)
                if borrow_mode == BorrowMode::Borrowed =>
            {
                program.emit(block, DecodeOp::ReadByteSliceRef { dst_offset });
                return Ok(());
            }
            _ => {}
        }
    }

    // Box<[T]> — fat pointer. Look up by structural shape identity.
    if let facet_core::Def::Slice(_) = pointee_shape.def {
        if let Some(cal) = cal
            && let Some(descriptor_handle) = cal.lookup_by_shape(shape)
        {
            let descriptor = OpaqueDescriptorId(descriptor_handle.0);
            let body_block = program.new_block();
            program.emit(
                block,
                DecodeOp::AllocBoxed {
                    dst_offset,
                    descriptor,
                    body_block,
                },
            );
            // Slice element decode is not yet implemented inline — slow path for now.
            program.emit(
                body_block,
                DecodeOp::SlowPath {
                    shape,
                    plan: Box::new(clone_plan(plan)),
                    dst_offset: 0,
                },
            );
            program.emit(body_block, DecodeOp::Return);
            return Ok(());
        }
        program.emit(
            block,
            DecodeOp::SlowPath {
                shape,
                plan: Box::new(clone_plan(plan)),
                dst_offset,
            },
        );
        return Ok(());
    }

    // Box<T> (non-slice). Look up by structural shape identity.
    if let Some(cal) = cal
        && let Some(descriptor_handle) = cal.lookup_by_shape(shape)
    {
        let descriptor = OpaqueDescriptorId(descriptor_handle.0);
        let body_block = program.new_block();
        program.emit(
            block,
            DecodeOp::AllocBoxed {
                dst_offset,
                descriptor,
                body_block,
            },
        );
        // Decode the pointee into the newly allocated slot (base = alloc ptr, offset = 0).
        lower_value(
            pointee_plan,
            pointee_shape,
            registry,
            Some(cal),
            borrow_mode,
            program,
            body_block,
            0,
        )?;
        program.emit(body_block, DecodeOp::Return);
        return Ok(());
    }

    // No calibration or no descriptor registered for this pointer shape — fall back.
    program.emit(
        block,
        DecodeOp::SlowPath {
            shape,
            plan: Box::new(clone_plan(plan)),
            dst_offset,
        },
    );
    Ok(())
}

fn build_identity_plan_for_struct(st: facet_core::StructType) -> TranslationPlan {
    let field_ops = (0..st.fields.len())
        .map(|i| FieldOp::Read { local_index: i })
        .collect();
    TranslationPlan::Struct {
        field_ops,
        nested: HashMap::new(),
    }
}

/// Shallow clone of a `TranslationPlan` for embedding in `SlowPath` ops.
fn clone_plan(plan: &TranslationPlan) -> TranslationPlan {
    match plan {
        TranslationPlan::Identity => TranslationPlan::Identity,
        TranslationPlan::Struct { field_ops, nested } => TranslationPlan::Struct {
            field_ops: field_ops.clone(),
            nested: nested.iter().map(|(&k, v)| (k, clone_plan(v))).collect(),
        },
        TranslationPlan::Enum {
            variant_map,
            variant_plans,
            nested,
        } => TranslationPlan::Enum {
            variant_map: variant_map.clone(),
            variant_plans: variant_plans
                .iter()
                .map(|(&k, v)| (k, clone_plan(v)))
                .collect(),
            nested: nested.iter().map(|(&k, v)| (k, clone_plan(v))).collect(),
        },
        TranslationPlan::Tuple { field_ops, nested } => TranslationPlan::Tuple {
            field_ops: field_ops.clone(),
            nested: nested.iter().map(|(&k, v)| (k, clone_plan(v))).collect(),
        },
        TranslationPlan::List { element } => TranslationPlan::List {
            element: Box::new(clone_plan(element)),
        },
        TranslationPlan::Option { inner } => TranslationPlan::Option {
            inner: Box::new(clone_plan(inner)),
        },
        TranslationPlan::Map { key, value } => TranslationPlan::Map {
            key: Box::new(clone_plan(key)),
            value: Box::new(clone_plan(value)),
        },
        TranslationPlan::Array { element } => TranslationPlan::Array {
            element: Box::new(clone_plan(element)),
        },
        TranslationPlan::Pointer { pointee } => TranslationPlan::Pointer {
            pointee: Box::new(clone_plan(pointee)),
        },
    }
}

// ---------------------------------------------------------------------------
// Pure IR interpreter                                         (Task #3)
// ---------------------------------------------------------------------------

/// Interpreter state — wraps the input cursor.
struct InterpState<'a> {
    input: &'a [u8],
    pos: usize,
    /// Scratch register for the last-decoded discriminant (enum dispatch).
    discriminant: u64,
    /// Scratch register for list length (used during list alloc/commit).
    list_len: usize,
}

impl<'a> InterpState<'a> {
    fn new(input: &'a [u8]) -> Self {
        Self {
            input,
            pos: 0,
            discriminant: 0,
            list_len: 0,
        }
    }

    fn read_byte(&mut self) -> Result<u8, DeserializeError> {
        if self.pos >= self.input.len() {
            return Err(DeserializeError::UnexpectedEof { pos: self.pos });
        }
        let b = self.input[self.pos];
        self.pos += 1;
        Ok(b)
    }

    fn read_bytes(&mut self, n: usize) -> Result<&'a [u8], DeserializeError> {
        if self.pos + n > self.input.len() {
            return Err(DeserializeError::UnexpectedEof { pos: self.pos });
        }
        let s = &self.input[self.pos..self.pos + n];
        self.pos += n;
        Ok(s)
    }

    fn read_varint(&mut self) -> Result<u64, DeserializeError> {
        let start = self.pos;
        let mut result: u64 = 0;
        let mut shift: u32 = 0;
        loop {
            let byte = self.read_byte()?;
            result |= ((byte & 0x7F) as u64) << shift;
            if byte & 0x80 == 0 {
                return Ok(result);
            }
            shift += 7;
            if shift >= 64 {
                return Err(DeserializeError::VarintOverflow { pos: start });
            }
        }
    }

    fn read_signed_varint(&mut self) -> Result<i64, DeserializeError> {
        let z = self.read_varint()?;
        Ok(((z >> 1) as i64) ^ (-((z & 1) as i64)))
    }

    fn read_varint_u128(&mut self) -> Result<u128, DeserializeError> {
        let start = self.pos;
        let mut result: u128 = 0;
        let mut shift: u32 = 0;
        loop {
            let byte = self.read_byte()?;
            result |= ((byte & 0x7F) as u128) << shift;
            if byte & 0x80 == 0 {
                return Ok(result);
            }
            shift += 7;
            if shift >= 128 {
                return Err(DeserializeError::VarintOverflow { pos: start });
            }
        }
    }

    fn read_signed_varint_i128(&mut self) -> Result<i128, DeserializeError> {
        let z = self.read_varint_u128()?;
        Ok(((z >> 1) as i128) ^ (-((z & 1) as i128)))
    }

    fn read_str(&mut self) -> Result<&'a str, DeserializeError> {
        let len = self.read_varint()? as usize;
        let bytes = self.read_bytes(len)?;
        std::str::from_utf8(bytes).map_err(|_| DeserializeError::InvalidUtf8 {
            pos: self.pos - len,
        })
    }

    fn read_byte_slice(&mut self) -> Result<&'a [u8], DeserializeError> {
        let len = self.read_varint()? as usize;
        self.read_bytes(len)
    }

    fn read_opaque_bytes(&mut self) -> Result<&'a [u8], DeserializeError> {
        let len_bytes = self.read_bytes(4)?;
        let len =
            u32::from_le_bytes([len_bytes[0], len_bytes[1], len_bytes[2], len_bytes[3]]) as usize;
        self.read_bytes(len)
    }
}

/// Interpret a `DecodeProgram` against `input`, writing the decoded value into
/// `dst` (which must point to at least `program.root_size` bytes of
/// writeable, properly-aligned memory, initialised to zero before calling).
///
/// `cal` may be `None` when calibration is unavailable; opaque fast-path ops
/// will fall back to conservative stubs in that case.
///
/// # Safety
///
/// - `dst` must be valid for writes of `program.root_size` bytes.
/// - `dst` must satisfy `program.root_align` alignment.
/// - Fields written by the program must not alias.
/// - The caller is responsible for dropping `dst` contents if this returns an
///   error after partial writes.
#[allow(unsafe_code)]
pub unsafe fn interpret(
    program: &DecodeProgram,
    input: &[u8],
    dst: *mut u8,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
) -> Result<usize, DeserializeError> {
    let mut state = InterpState::new(input);
    run_block(program, 0, &mut state, dst, registry, cal)?;
    Ok(state.pos)
}

#[allow(unsafe_code)]
fn run_block(
    program: &DecodeProgram,
    block_id: usize,
    state: &mut InterpState<'_>,
    base: *mut u8,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
) -> Result<(), DeserializeError> {
    let block = &program.blocks[block_id];
    let mut i = 0;
    while i < block.ops.len() {
        let op = &block.ops[i];
        match op {
            DecodeOp::ReadScalar { prim, dst_offset } => {
                let dst = unsafe { base.add(*dst_offset) };
                exec_read_scalar(state, *prim, dst)?;
            }

            DecodeOp::ReadFixedVec {
                dst_offset,
                descriptor,
                elem_size,
                validate_bool,
            } => {
                // Bulk-copy decode for `Vec<T>` where T's wire format is
                // bit-identical to its in-memory representation: read the
                // varint length, alloc an aligned backing of `len * elem_size`
                // bytes, memcpy from input, write the container header.
                let cal = cal.ok_or_else(|| {
                    DeserializeError::Custom("ReadFixedVec requires a calibration registry".into())
                })?;
                let desc = cal
                    .get(vox_jit_cal::DescriptorHandle(descriptor.0))
                    .ok_or_else(|| {
                        DeserializeError::Custom("ReadFixedVec: descriptor handle not found".into())
                    })?;
                let list_len = state.read_varint()? as usize;
                let byte_count = list_len * *elem_size;

                if list_len == 0 {
                    unsafe {
                        std::ptr::copy_nonoverlapping(
                            desc.empty_bytes.as_ptr(),
                            base.add(*dst_offset),
                            desc.empty_bytes.len(),
                        );
                    }
                } else {
                    let layout = std::alloc::Layout::from_size_align(byte_count, desc.elem_align)
                        .map_err(|_| {
                        DeserializeError::Custom("ReadFixedVec: invalid layout".into())
                    })?;
                    let backing_ptr = unsafe { std::alloc::alloc(layout) };
                    if backing_ptr.is_null() {
                        return Err(DeserializeError::Custom(
                            "ReadFixedVec: allocation failed (OOM)".into(),
                        ));
                    }
                    let src_bytes = state.read_bytes(byte_count)?;
                    unsafe {
                        std::ptr::copy_nonoverlapping(src_bytes.as_ptr(), backing_ptr, byte_count);
                    }
                    if *validate_bool {
                        let mut acc: u8 = 0;
                        for &b in src_bytes {
                            acc |= b;
                        }
                        if acc > 1 {
                            unsafe { std::alloc::dealloc(backing_ptr, layout) };
                            return Err(DeserializeError::Custom(
                                "ReadFixedVec: invalid bool byte (must be 0 or 1)".into(),
                            ));
                        }
                    }
                    let dst = unsafe { base.add(*dst_offset) };
                    unsafe {
                        std::ptr::write(
                            dst.add(desc.ptr_offset as usize) as *mut *mut u8,
                            backing_ptr,
                        );
                        std::ptr::write(dst.add(desc.len_offset as usize) as *mut usize, list_len);
                        std::ptr::write(dst.add(desc.cap_offset as usize) as *mut usize, list_len);
                    }
                }
            }

            DecodeOp::ReadString {
                dst_offset,
                descriptor: _,
            } => {
                let s = state.read_str()?;
                let owned = s.to_owned();
                unsafe {
                    std::ptr::write(base.add(*dst_offset) as *mut String, owned);
                }
            }

            DecodeOp::ReadCowStr {
                dst_offset,
                borrowed,
            } => {
                let s = state.read_str()?;
                let dst = unsafe { base.add(*dst_offset) as *mut std::borrow::Cow<'static, str> };
                let value = if *borrowed {
                    let borrowed: &'static str = unsafe { std::mem::transmute(s) };
                    std::borrow::Cow::Borrowed(borrowed)
                } else {
                    std::borrow::Cow::Owned(s.to_owned())
                };
                unsafe {
                    std::ptr::write(dst, value);
                }
            }

            DecodeOp::ReadStrRef { dst_offset } => {
                let s = state.read_str()?;
                let s: &'static str = unsafe { std::mem::transmute(s) };
                unsafe {
                    std::ptr::write(base.add(*dst_offset) as *mut &'static str, s);
                }
            }

            DecodeOp::ReadCowByteSlice {
                dst_offset,
                borrowed,
            } => {
                let bytes = state.read_byte_slice()?;
                let dst = unsafe { base.add(*dst_offset) as *mut std::borrow::Cow<'static, [u8]> };
                let value = if *borrowed {
                    let borrowed: &'static [u8] = unsafe { std::mem::transmute(bytes) };
                    std::borrow::Cow::Borrowed(borrowed)
                } else {
                    std::borrow::Cow::Owned(bytes.to_vec())
                };
                unsafe {
                    std::ptr::write(dst, value);
                }
            }

            DecodeOp::ReadByteSliceRef { dst_offset } => {
                let bytes = state.read_byte_slice()?;
                let bytes: &'static [u8] = unsafe { std::mem::transmute(bytes) };
                unsafe {
                    std::ptr::write(base.add(*dst_offset) as *mut &'static [u8], bytes);
                }
            }

            DecodeOp::ReadOpaque { shape, dst_offset } => {
                let bytes = state.read_opaque_bytes()?;
                let adapter = shape.opaque_adapter.ok_or_else(|| {
                    DeserializeError::ReflectError(format!("missing opaque adapter for {shape}"))
                })?;
                let input = facet::OpaqueDeserialize::Borrowed(bytes);
                unsafe {
                    (adapter.deserialize)(input, facet_core::PtrUninit::new(base.add(*dst_offset)))
                }
                .map_err(|e| {
                    DeserializeError::ReflectError(format!(
                        "opaque adapter deserialize failed for {shape}: {e}"
                    ))
                })?;
            }

            DecodeOp::SkipValue { kind } => {
                skip_in_state(state, kind, registry)?;
            }

            DecodeOp::WriteDefault { shape, dst_offset } => {
                let dst = unsafe { base.add(*dst_offset) };
                unsafe {
                    shape
                        .call_default_in_place(facet_core::PtrUninit::new(dst as *mut ()))
                        .ok_or_else(|| {
                            DeserializeError::ReflectError(format!(
                                "no Default available for fill-defaults field {shape}"
                            ))
                        })?;
                }
            }

            DecodeOp::DecodeOption {
                dst_offset,
                inner_offset,
                some_block,
                none_bytes,
                some_bytes,
            } => {
                let tag = state.read_byte()?;
                let option_ptr = unsafe { base.add(*dst_offset) };
                match tag {
                    0x00 => unsafe {
                        std::ptr::copy_nonoverlapping(
                            none_bytes.as_ptr(),
                            option_ptr,
                            none_bytes.len(),
                        );
                    },
                    0x01 => {
                        unsafe {
                            std::ptr::copy_nonoverlapping(
                                some_bytes.as_ptr(),
                                option_ptr,
                                some_bytes.len(),
                            );
                        }
                        let inner_ptr = unsafe { option_ptr.add(*inner_offset) };
                        run_block(program, *some_block, state, inner_ptr, registry, cal)?;
                    }
                    other => {
                        return Err(DeserializeError::InvalidOptionTag {
                            pos: state.pos - 1,
                            got: other,
                        });
                    }
                }
            }

            DecodeOp::DecodeResult {
                dst_offset,
                ok_block,
                err_block,
                ok_offset,
                err_offset,
                ok_bytes,
                err_bytes,
            } => {
                let variant_index = state.read_varint()? as usize;
                let result_ptr = unsafe { base.add(*dst_offset) };
                match variant_index {
                    0 => {
                        unsafe {
                            std::ptr::copy_nonoverlapping(
                                ok_bytes.as_ptr(),
                                result_ptr,
                                ok_bytes.len(),
                            );
                        }
                        let payload_ptr = unsafe { result_ptr.add(*ok_offset) };
                        run_block(program, *ok_block, state, payload_ptr, registry, cal)?;
                    }
                    1 => {
                        unsafe {
                            std::ptr::copy_nonoverlapping(
                                err_bytes.as_ptr(),
                                result_ptr,
                                err_bytes.len(),
                            );
                        }
                        let payload_ptr = unsafe { result_ptr.add(*err_offset) };
                        run_block(program, *err_block, state, payload_ptr, registry, cal)?;
                    }
                    other => {
                        return Err(DeserializeError::UnknownVariant {
                            remote_index: other,
                        });
                    }
                }
            }

            DecodeOp::DecodeResultInit {
                dst_offset,
                ok_block,
                err_block,
                ok_size,
                ok_align,
                err_size,
                err_align,
                init_ok_fn,
                init_err_fn,
            } => {
                let variant_index = state.read_varint()? as usize;
                let result_ptr = unsafe { base.add(*dst_offset) };
                match variant_index {
                    0 => {
                        let layout = std::alloc::Layout::from_size_align(*ok_size, *ok_align)
                            .map_err(|_| {
                                DeserializeError::Custom("bad Result::Ok layout".into())
                            })?;
                        let tmp = facet_core::alloc_for_layout(layout);
                        let tmp_ptr = unsafe { tmp.assume_init() };
                        if let Err(err) = run_block(
                            program,
                            *ok_block,
                            state,
                            tmp_ptr.as_mut_byte_ptr(),
                            registry,
                            cal,
                        ) {
                            unsafe { facet_core::dealloc_for_layout(tmp_ptr, layout) };
                            return Err(err);
                        }
                        unsafe {
                            init_ok_fn(facet_core::PtrUninit::new(result_ptr), tmp_ptr);
                            facet_core::dealloc_for_layout(tmp_ptr, layout);
                        }
                    }
                    1 => {
                        let layout = std::alloc::Layout::from_size_align(*err_size, *err_align)
                            .map_err(|_| {
                                DeserializeError::Custom("bad Result::Err layout".into())
                            })?;
                        let tmp = facet_core::alloc_for_layout(layout);
                        let tmp_ptr = unsafe { tmp.assume_init() };
                        if let Err(err) = run_block(
                            program,
                            *err_block,
                            state,
                            tmp_ptr.as_mut_byte_ptr(),
                            registry,
                            cal,
                        ) {
                            unsafe { facet_core::dealloc_for_layout(tmp_ptr, layout) };
                            return Err(err);
                        }
                        unsafe {
                            init_err_fn(facet_core::PtrUninit::new(result_ptr), tmp_ptr);
                            facet_core::dealloc_for_layout(tmp_ptr, layout);
                        }
                    }
                    other => {
                        return Err(DeserializeError::UnknownVariant {
                            remote_index: other,
                        });
                    }
                }
            }

            DecodeOp::ReadDiscriminant => {
                state.discriminant = state.read_varint()?;
            }

            // r[impl schema.errors.unknown-variant-runtime]
            DecodeOp::BranchOnVariant {
                tag_offset,
                tag_width,
                variant_table,
                variant_blocks,
            } => {
                let remote_disc = state.discriminant as usize;
                let local_idx = variant_table.get(remote_disc).copied().flatten().ok_or(
                    DeserializeError::UnknownVariant {
                        remote_index: remote_disc,
                    },
                )?;

                // There may be no corresponding entry in variant_blocks if the
                // remote had more variants than we mapped.
                let (local_disc, variant_block) = variant_blocks
                    .get(remote_disc)
                    .copied()
                    .filter(|&(_, b)| b != usize::MAX)
                    .ok_or(DeserializeError::UnknownVariant {
                        remote_index: remote_disc,
                    })?;

                // Write the local discriminant tag
                write_tag(unsafe { base.add(*tag_offset) }, *tag_width, local_disc);

                run_block(program, variant_block, state, base, registry, cal)?;

                let _ = local_idx; // used implicitly via variant_block selection
            }

            DecodeOp::PushFrame {
                field_offset,
                frame_size: _,
            } => {
                let new_base = unsafe { base.add(*field_offset) };
                // Execute the rest of the current block list from the new base?
                // PushFrame/PopFrame would normally be paired; for now, since the
                // interpreter uses `run_block` recursion for nested types, we
                // don't need an explicit stack here.  This op is reserved for the
                // Cranelift backend where explicit frame management is needed.
                let _ = new_base;
            }

            DecodeOp::PopFrame => { /* see PushFrame — no-op in interpreter */ }

            DecodeOp::ReadListLen {
                descriptor: _,
                dst_offset: _,
                empty_block,
                body_block,
            } => {
                let len = state.read_varint()? as usize;
                state.list_len = len;
                if len == 0 {
                    run_block(program, *empty_block, state, base, registry, cal)?;
                } else {
                    run_block(program, *body_block, state, base, registry, cal)?;
                }
            }

            DecodeOp::CommitListLen {
                dst_offset: _,
                descriptor: _,
            } => {
                // No-op in the interpreter: by the time `inner_block` runs,
                // `base` is the element pointer (not the container header),
                // so `dst_offset` here would be wrong. The interpreter's
                // `AllocBacking` handler commits the running len directly
                // against the outer base after each iteration. The op
                // remains in the program because the JIT also skips it
                // inline (`emit_alloc_backing`'s loop tail does the store).
            }

            DecodeOp::DecodeArray {
                dst_offset,
                count,
                elem_size,
                body_block,
            } => {
                let mut elem_ptr = unsafe { base.add(*dst_offset) };
                for _ in 0..*count {
                    run_block(program, *body_block, state, elem_ptr, registry, cal)?;
                    elem_ptr = unsafe { elem_ptr.add(*elem_size) };
                }
            }

            DecodeOp::MaterializeEmpty {
                dst_offset,
                descriptor,
            } => {
                // Use the calibrated empty bytes when available; fall back to
                // zeroing the slot (conservative but correct on all current
                // Rust targets where Vec/String empty repr is all-zeros).
                if let Some(cal) = cal
                    && let Some(desc) = cal.get(DescriptorHandle::from(*descriptor))
                {
                    unsafe {
                        std::ptr::copy_nonoverlapping(
                            desc.empty_bytes.as_ptr(),
                            base.add(*dst_offset),
                            desc.empty_bytes.len(),
                        );
                    }
                } else {
                    unsafe {
                        std::ptr::write_bytes(
                            base.add(*dst_offset),
                            0,
                            std::mem::size_of::<usize>() * 3,
                        );
                    }
                }
            }

            DecodeOp::AllocBacking {
                dst_offset,
                descriptor,
                body_block,
                elem_size,
            } => {
                // Allocate the backing store, write the container header,
                // then drive the element loop. Mirrors the JIT codegen in
                // `emit_alloc_backing` so the two paths produce identical
                // values for the same input bytes.
                let cal = cal.ok_or_else(|| {
                    DeserializeError::Custom("AllocBacking requires a calibration registry".into())
                })?;
                let desc = cal
                    .get(vox_jit_cal::DescriptorHandle(descriptor.0))
                    .ok_or_else(|| {
                        DeserializeError::Custom("AllocBacking: descriptor handle not found".into())
                    })?;

                let list_len = state.list_len;
                let backing_ptr: *mut u8 = if desc.elem_size == 0 || list_len == 0 {
                    desc.elem_align as *mut u8
                } else {
                    let layout = std::alloc::Layout::from_size_align(
                        list_len * desc.elem_size,
                        desc.elem_align,
                    )
                    .map_err(|_| DeserializeError::Custom("AllocBacking: invalid layout".into()))?;
                    let p = unsafe { std::alloc::alloc(layout) };
                    if p.is_null() {
                        return Err(DeserializeError::Custom(
                            "AllocBacking: allocation failed (OOM)".into(),
                        ));
                    }
                    p
                };

                // Write ptr/len/cap into the container header. Start `len`
                // at zero — `CommitListLen` bumps it after each element so
                // a mid-loop error leaves a valid partial container.
                let dst = unsafe { base.add(*dst_offset) };
                unsafe {
                    std::ptr::write(
                        dst.add(desc.ptr_offset as usize) as *mut *mut u8,
                        backing_ptr,
                    );
                    if desc.len_offset != vox_jit_cal::OFFSET_ABSENT {
                        std::ptr::write(dst.add(desc.len_offset as usize) as *mut usize, 0);
                    }
                    if desc.cap_offset != vox_jit_cal::OFFSET_ABSENT {
                        std::ptr::write(dst.add(desc.cap_offset as usize) as *mut usize, list_len);
                    }
                }

                // Save outer loop state so nested ReadListLen/AllocBacking
                // (e.g. Vec<Vec<u8>>) can use their own list_len without
                // corrupting ours.
                let outer_list_len = state.list_len;
                let len_slot = if desc.len_offset != vox_jit_cal::OFFSET_ABSENT {
                    Some(unsafe { dst.add(desc.len_offset as usize) as *mut usize })
                } else {
                    None
                };
                for i in 0..list_len {
                    let elem_ptr = unsafe { backing_ptr.add(i * elem_size) };
                    run_block(program, *body_block, state, elem_ptr, registry, cal.into())?;
                    // Commit len after each element: a mid-loop error then
                    // leaves a valid partial container (drop runs on the
                    // initialized prefix only).
                    if let Some(ptr) = len_slot {
                        unsafe { std::ptr::write(ptr, i + 1) };
                    }
                }
                state.list_len = outer_list_len;
            }

            DecodeOp::AllocBoxed {
                dst_offset,
                descriptor,
                body_block,
            } => {
                let Some(cal) = cal else {
                    return Err(DeserializeError::Custom(
                        "AllocBoxed requires a calibration registry".into(),
                    ));
                };
                let handle = DescriptorHandle(descriptor.0);
                let desc = cal.get(handle).ok_or_else(|| {
                    DeserializeError::Custom("AllocBoxed: descriptor handle not found".into())
                })?;
                let alloc_ptr = if desc.elem_size == 0 {
                    desc.elem_align as *mut u8
                } else {
                    let layout =
                        std::alloc::Layout::from_size_align(desc.elem_size, desc.elem_align)
                            .map_err(|_| {
                                DeserializeError::Custom("AllocBoxed: invalid layout".into())
                            })?;
                    let p = unsafe { std::alloc::alloc(layout) };
                    if p.is_null() {
                        return Err(DeserializeError::Custom(
                            "AllocBoxed: allocation failed (OOM)".into(),
                        ));
                    }
                    p
                };
                let container_ptr = unsafe { base.add(*dst_offset) };
                unsafe {
                    let ptr_slot = container_ptr.add(desc.ptr_offset as usize) as *mut *mut u8;
                    std::ptr::write(ptr_slot, alloc_ptr);
                }
                run_block(program, *body_block, state, alloc_ptr, registry, Some(cal))?;
            }

            DecodeOp::DecodeMap {
                dst_offset,
                from_pair_slice,
                pair_stride,
                pair_align,
                value_offset: _,
                body_block,
            } => {
                // Slab strategy: decode every pair into a contiguous scratch
                // buffer, then build the map in one `from_pair_slice` call.
                // Mirrors `emit_decode_map` in the JIT codegen so the two
                // engines produce identical maps for the same input bytes.
                let len = state.read_varint()? as usize;
                let stride = *pair_stride;

                // Stride 0 (both K and V are ZSTs) or an empty map needs no
                // allocation: `from_pair_slice` reads `len` ZST pairs (or
                // none) and never dereferences a real address.
                let (slab, slab_layout): (*mut u8, Option<std::alloc::Layout>) =
                    if len == 0 || stride == 0 {
                        (*pair_align as *mut u8, None)
                    } else {
                        let size = len.checked_mul(stride).ok_or_else(|| {
                            DeserializeError::Custom("DecodeMap: slab size overflow".into())
                        })?;
                        let layout = std::alloc::Layout::from_size_align(size, *pair_align)
                            .map_err(|_| {
                                DeserializeError::Custom("DecodeMap: invalid slab layout".into())
                            })?;
                        let p = unsafe { std::alloc::alloc(layout) };
                        if p.is_null() {
                            return Err(DeserializeError::Custom(
                                "DecodeMap: slab allocation failed (OOM)".into(),
                            ));
                        }
                        (p, Some(layout))
                    };

                // Decode each `(K, V)` pair into its slab slot. The body block
                // writes the key at slot+0 and the value at slot+value_offset.
                let mut decode_result = Ok(());
                for i in 0..len {
                    let slot = unsafe { slab.add(i * stride) };
                    if let Err(e) = run_block(program, *body_block, state, slot, registry, cal) {
                        decode_result = Err(e);
                        break;
                    }
                }

                match decode_result {
                    Ok(()) => {
                        // Build the map from the fully-populated slab. This
                        // moves every `(K, V)` out of the slab via `ptr::read`,
                        // leaving the buffer logically uninitialized.
                        let from_pair_slice = *from_pair_slice;
                        let dst = unsafe { base.add(*dst_offset) };
                        unsafe {
                            from_pair_slice(facet_core::PtrUninit::new(dst as *mut ()), slab, len);
                        }
                        if let Some(layout) = slab_layout {
                            unsafe { std::alloc::dealloc(slab, layout) };
                        }
                    }
                    Err(e) => {
                        // Malformed input mid-decode: free the slab buffer.
                        // The keys/values decoded so far leak their contents —
                        // consistent with the partial-value leak model the
                        // decode path already has on malformed input.
                        if let Some(layout) = slab_layout {
                            unsafe { std::alloc::dealloc(slab, layout) };
                        }
                        return Err(e);
                    }
                }
            }

            DecodeOp::SlowPath {
                shape,
                plan,
                dst_offset,
            } => {
                exec_slow_path(state, shape, plan, *dst_offset, base, registry)?;
            }

            DecodeOp::Jump { block_id } => {
                return run_block(program, *block_id, state, base, registry, cal);
            }

            DecodeOp::Return => {
                return Ok(());
            }

            DecodeOp::CallSelf { dst_offset } | DecodeOp::TailCallSelf { dst_offset } => {
                let new_base = unsafe { base.add(*dst_offset) };
                run_block(program, 0, state, new_base, registry, cal)?;
            }
        }

        i += 1;
    }

    Ok(())
}

/// Scalar write helper — writes a decoded value at `dst` (correctly typed).
#[allow(unsafe_code)]
fn exec_read_scalar(
    state: &mut InterpState<'_>,
    prim: WirePrimitive,
    dst: *mut u8,
) -> Result<(), DeserializeError> {
    match prim {
        WirePrimitive::Unit => {}
        WirePrimitive::Bool => {
            let b = state.read_byte()?;
            match b {
                0x00 => unsafe { std::ptr::write(dst as *mut bool, false) },
                0x01 => unsafe { std::ptr::write(dst as *mut bool, true) },
                other => {
                    return Err(DeserializeError::InvalidBool {
                        pos: state.pos - 1,
                        got: other,
                    });
                }
            }
        }
        WirePrimitive::U8 => {
            let v = state.read_byte()?;
            unsafe { std::ptr::write(dst, v) };
        }
        WirePrimitive::U16 => {
            let v = state.read_varint()? as u16;
            unsafe { std::ptr::write(dst as *mut u16, v) };
        }
        WirePrimitive::U32 => {
            let v = state.read_varint()? as u32;
            unsafe { std::ptr::write(dst as *mut u32, v) };
        }
        WirePrimitive::U64 => {
            let v = state.read_varint()?;
            unsafe { std::ptr::write(dst as *mut u64, v) };
        }
        WirePrimitive::U128 => {
            let v = state.read_varint_u128()?;
            unsafe { std::ptr::write(dst as *mut u128, v) };
        }
        WirePrimitive::USize => {
            let v = state.read_varint()? as usize;
            unsafe { std::ptr::write(dst as *mut usize, v) };
        }
        WirePrimitive::I8 => {
            let v = state.read_byte()? as i8;
            unsafe { std::ptr::write(dst as *mut i8, v) };
        }
        WirePrimitive::I16 => {
            let v = state.read_signed_varint()? as i16;
            unsafe { std::ptr::write(dst as *mut i16, v) };
        }
        WirePrimitive::I32 => {
            let v = state.read_signed_varint()? as i32;
            unsafe { std::ptr::write(dst as *mut i32, v) };
        }
        WirePrimitive::I64 => {
            let v = state.read_signed_varint()?;
            unsafe { std::ptr::write(dst as *mut i64, v) };
        }
        WirePrimitive::I128 => {
            let v = state.read_signed_varint_i128()?;
            unsafe { std::ptr::write(dst as *mut i128, v) };
        }
        WirePrimitive::ISize => {
            let v = state.read_signed_varint()? as isize;
            unsafe { std::ptr::write(dst as *mut isize, v) };
        }
        WirePrimitive::F32 => {
            let bytes = state.read_bytes(4)?;
            let v = f32::from_le_bytes(bytes.try_into().unwrap());
            unsafe { std::ptr::write(dst as *mut f32, v) };
        }
        WirePrimitive::F64 => {
            let bytes = state.read_bytes(8)?;
            let v = f64::from_le_bytes(bytes.try_into().unwrap());
            unsafe { std::ptr::write(dst as *mut f64, v) };
        }
        WirePrimitive::String => {
            let s = state.read_str()?;
            let owned = s.to_owned();
            unsafe { std::ptr::write(dst as *mut String, owned) };
        }
        WirePrimitive::Bytes => {
            let bytes = state.read_byte_slice()?;
            let vec: Vec<u8> = bytes.to_vec();
            unsafe { std::ptr::write(dst as *mut Vec<u8>, vec) };
        }
        WirePrimitive::Payload => {
            let len_bytes = state.read_bytes(4)?;
            let len = u32::from_le_bytes(len_bytes.try_into().unwrap()) as usize;
            let payload = state.read_bytes(len)?.to_vec();
            unsafe { std::ptr::write(dst as *mut Vec<u8>, payload) };
        }
        WirePrimitive::Char => {
            let s = state.read_str()?;
            let c = s
                .chars()
                .next()
                .ok_or_else(|| DeserializeError::Custom("empty string for char".into()))?;
            unsafe { std::ptr::write(dst as *mut char, c) };
        }
    }
    Ok(())
}

/// Write `disc` into `tag_ptr` according to `width`.
#[allow(unsafe_code)]
fn write_tag(tag_ptr: *mut u8, width: TagWidth, disc: u64) {
    match width {
        TagWidth::U8 => unsafe { std::ptr::write(tag_ptr, disc as u8) },
        TagWidth::U16 => unsafe { std::ptr::write(tag_ptr as *mut u16, disc as u16) },
        TagWidth::U32 => unsafe { std::ptr::write(tag_ptr as *mut u32, disc as u32) },
        TagWidth::U64 => unsafe { std::ptr::write(tag_ptr as *mut u64, disc) },
    }
}

/// Skip a value using the existing `decode::skip_value` path.
/// We rebuild a temporary cursor from the current state position.
fn skip_in_state(
    state: &mut InterpState<'_>,
    kind: &SchemaKind,
    registry: &SchemaRegistry,
) -> Result<(), DeserializeError> {
    let mut tmp = crate::decode::Cursor::new(state.input);
    tmp.advance_to(state.pos);
    crate::decode::skip_value(&mut tmp, kind, registry)?;
    state.pos = tmp.pos();
    Ok(())
}

// r[impl schema.exchange.required]
/// Slow-path: fall back to the reflective `deserialize_value` path for
/// shapes the IR could not lower.
///
/// Creates a `Partial` over the caller-provided destination slot (no extra
/// allocation), deserializes into it in-place, and calls `finish_in_place`
/// so that optional-field defaults are applied.
#[allow(unsafe_code)]
fn exec_slow_path(
    state: &mut InterpState<'_>,
    shape: &'static Shape,
    plan: &TranslationPlan,
    dst_offset: usize,
    base: *mut u8,
    registry: &SchemaRegistry,
) -> Result<(), DeserializeError> {
    use facet_core::PtrUninit;
    use facet_reflect::Partial;

    let remaining = &state.input[state.pos..];
    let mut cursor = crate::decode::Cursor::new(remaining);

    let dst_ptr = unsafe { base.add(dst_offset) };
    let uninit = PtrUninit::new(dst_ptr as *mut ());
    let partial = unsafe { Partial::from_raw_with_shape(uninit, shape) }
        .map_err(|e| DeserializeError::ReflectError(e.to_string()))?;

    let partial =
        crate::deserialize::deserialize_value_pub::<false>(partial, &mut cursor, plan, registry)?;

    partial
        .finish_in_place()
        .map_err(|e| DeserializeError::ReflectError(e.to_string()))?;

    state.pos += cursor.pos();
    Ok(())
}

/// Public raw bridge for the JIT SlowPath helper.
///
/// Decodes a single value of `shape` using the reflective interpreter, reading
/// from `input[consumed..]`. On success writes the initialized value into
/// `dst_base.add(dst_offset)` and returns the new consumed position.
/// On failure returns `None`.
///
/// A fresh `SchemaRegistry` is used — SlowPath types use identity plans that
/// contain no `FieldOp::Skip` entries, so no cross-schema lookups are needed.
///
/// # Safety
///
/// - `input_ptr` must be valid for reads of `input_len` bytes.
/// - `consumed` must be less than or equal to `input_len`.
/// - `plan` must be a valid, non-null pointer to a `TranslationPlan` that
///   remains valid for the duration of this call.
/// - `dst_base` must be valid for writes of at least `dst_offset +
///   size_of::<T>()` bytes, where `T` is the type described by `shape`.
/// - `dst_base.add(dst_offset)` must be properly aligned for the type
///   described by `shape`.
/// - The memory at `dst_base.add(dst_offset)` must be uninitialized or
///   otherwise safe to overwrite (the caller is responsible for dropping
///   any previously initialized value at that location).
pub unsafe fn slow_path_decode_raw(
    input_ptr: *const u8,
    input_len: usize,
    consumed: usize,
    shape: &'static Shape,
    plan: *const TranslationPlan,
    dst_base: *mut u8,
    dst_offset: usize,
) -> Option<usize> {
    use facet_core::PtrUninit;
    use facet_reflect::Partial;

    let input = unsafe { core::slice::from_raw_parts(input_ptr, input_len) };
    let remaining = &input[consumed..];
    let mut cursor = crate::decode::Cursor::new(remaining);

    let dst_ptr = unsafe { dst_base.add(dst_offset) };
    let uninit = PtrUninit::new(dst_ptr as *mut ());
    let partial = unsafe { Partial::from_raw_with_shape(uninit, shape) }.ok()?;

    let plan_ref = unsafe { &*plan };
    let registry = vox_schema::SchemaRegistry::new();

    let partial = crate::deserialize::deserialize_value_pub::<false>(
        partial,
        &mut cursor,
        plan_ref,
        &registry,
    )
    .ok()?;

    partial.finish_in_place().ok()?;

    Some(consumed + cursor.pos())
}

// ---------------------------------------------------------------------------
// Public entry point for IR-based deserialization
// ---------------------------------------------------------------------------

// r[impl schema.translation.serialization-unchanged]
/// Deserialize `input` into a value of type `T` using the IR interpreter.
///
/// Falls back to `SlowPath` ops for shapes not yet handled by the IR lowering.
/// This is the correctness oracle path — it must agree with the reflective
/// interpreter for all valid inputs.
///
/// Pass `cal` to enable the calibrated fast path for opaque types (`Vec<T>`,
/// `String`). Pass `None` to use zero-filled fallbacks for those ops.
pub fn from_slice_ir<T>(
    input: &[u8],
    plan: &TranslationPlan,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
) -> Result<T, DeserializeError>
where
    T: facet::Facet<'static>,
{
    unsafe { from_slice_ir_impl::<T>(input, plan, registry, cal, BorrowMode::Owned) }
}

/// Borrowed-mode sibling of [`from_slice_ir`]. Emits `ReadStrRef` /
/// `ReadByteSliceRef` / borrowed `Cow` ops so the decoded value may hold
/// references into `input`.
///
/// The `'input: 'facet` bound mirrors the JIT `try_decode_borrowed` entry
/// point — the input must outlive the borrowed value.
pub fn from_slice_ir_borrowed<'input, 'facet, T>(
    input: &'input [u8],
    plan: &TranslationPlan,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
) -> Result<T, DeserializeError>
where
    T: facet::Facet<'facet>,
    'input: 'facet,
{
    unsafe { from_slice_ir_impl::<T>(input, plan, registry, cal, BorrowMode::Borrowed) }
}

/// SAFETY: caller must uphold the lifetime contract between `input` and `T`
/// when `borrow_mode` is `Borrowed` — the `BorrowMode::Owned` public wrapper
/// enforces `T: Facet<'static>`; the `Borrowed` wrapper enforces
/// `'input: 'facet`.
#[allow(unsafe_code)]
unsafe fn from_slice_ir_impl<'facet, T>(
    input: &[u8],
    plan: &TranslationPlan,
    registry: &SchemaRegistry,
    cal: Option<&CalibrationRegistry>,
    borrow_mode: BorrowMode,
) -> Result<T, DeserializeError>
where
    T: facet::Facet<'facet>,
{
    let shape = T::SHAPE;
    let program = lower_with_cal(plan, shape, registry, cal, borrow_mode).map_err(|e| match e {
        LowerError::UnsizedShape => DeserializeError::UnsupportedType("unsized shape".into()),
        LowerError::UnstableEnumRepr => {
            DeserializeError::UnsupportedType("unstable enum repr".into())
        }
        LowerError::SchemaMissing => DeserializeError::Custom("schema missing during lower".into()),
        LowerError::Unsupported(reason) => DeserializeError::UnsupportedType(reason),
    })?;

    let layout = shape
        .layout
        .sized_layout()
        .map_err(|_| DeserializeError::UnsupportedType(format!("{shape}")))?;

    let ptr = {
        let p = unsafe {
            std::alloc::alloc_zeroed(
                std::alloc::Layout::from_size_align(layout.size(), layout.align())
                    .map_err(|_| DeserializeError::Custom("bad layout".into()))?,
            )
        };
        if p.is_null() {
            std::alloc::handle_alloc_error(
                std::alloc::Layout::from_size_align(layout.size(), layout.align()).unwrap(),
            );
        }
        p
    };

    let result = unsafe { interpret(&program, input, ptr, registry, cal) };

    match result {
        Ok(_bytes_consumed) => {
            let value: T = unsafe { std::ptr::read(ptr as *const T) };
            unsafe {
                std::alloc::dealloc(
                    ptr,
                    std::alloc::Layout::from_size_align(layout.size(), layout.align()).unwrap(),
                );
            }
            Ok(value)
        }
        Err(e) => {
            unsafe {
                std::ptr::write_bytes(ptr, 0, layout.size());
                std::alloc::dealloc(
                    ptr,
                    std::alloc::Layout::from_size_align(layout.size(), layout.align()).unwrap(),
                );
            }
            Err(e)
        }
    }
}

// ---------------------------------------------------------------------------
// Encode IR: EncodeOp + EncodeProgram + lower_encode           (Task #17)
// ---------------------------------------------------------------------------
//
// Encode does NOT use translation plans. It walks the sender's local type
// layout directly and writes postcard bytes into an EncodeCtx buffer.
//
// The IR is symmetric to DecodeProgram but simpler: no skip operations,
// no discriminant mapping, no partial-init concerns.

/// A single IR instruction for the encode path.
///
/// Instructions read from an implicit source pointer (`*const u8` base +
/// field offsets) and write bytes to an implicit output buffer (EncodeCtx).
///
/// Operand convention:
///   `src_offset`: byte offset from the base of the struct being read.
#[derive(Debug, Clone)]
pub enum EncodeOp {
    // -----------------------------------------------------------------------
    // Primitive writes
    // -----------------------------------------------------------------------
    /// Read a scalar primitive from `src_offset` and write its postcard
    /// encoding to the output buffer.
    WriteScalar {
        prim: WirePrimitive,
        src_offset: usize,
    },

    /// Encode a string-like field (`String`, `&str`, `Cow<str>`) without the
    /// reflective walker.
    WriteStringLike {
        shape: &'static Shape,
        src_offset: usize,
    },

    /// Encode a bytes-like field (`Cow<[u8]>`, `&[u8]`) without the
    /// reflective walker.
    WriteBytesLike {
        shape: &'static Shape,
        src_offset: usize,
    },

    /// Encode a field by delegating to a nested encoder for the exact shape.
    WriteShape {
        shape: &'static Shape,
        src_offset: usize,
    },

    /// Encode an opaque-adapter field (`#[facet(opaque = ...)]`) via a dedicated
    /// helper that preserves postcard's length-prefixed opaque semantics while
    /// delegating nested value encoding back through the JIT runtime.
    WriteOpaque {
        shape: &'static Shape,
        src_offset: usize,
    },

    /// Encode a proxy field (`#[facet(proxy = ...)]`) by converting to the
    /// proxy value and then delegating nested encoding back through the JIT
    /// runtime.
    WriteProxy {
        shape: &'static Shape,
        src_offset: usize,
    },

    /// Encode one field reflectively from `src_offset`.
    SlowPath {
        shape: &'static Shape,
        src_offset: usize,
    },

    /// Borrow a pointee from a pointer-like value and continue encoding from
    /// the borrowed pointee pointer.
    BorrowPointer {
        src_offset: usize,
        body_block: usize,
        borrow_fn: facet_core::BorrowFn,
    },

    /// Like `BorrowPointer` but for pointer types whose in-memory layout is
    /// exactly a `*const T` (e.g. `Box<T>` for sized `T`). Codegen emits a
    /// single load instead of an indirect call to a vtable `borrow_fn`. The
    /// body block is emitted inline with the loaded pointer as the new
    /// `src_ptr` base.
    DerefPointer {
        src_offset: usize,
        body_block: usize,
    },

    /// Write a varint-length-prefixed byte slice from a slice-like value.
    WriteByteSlice {
        src_offset: usize,
        len_fn: facet_core::SliceLenFn,
        as_ptr_fn: facet_core::SliceAsPtrFn,
    },

    // -----------------------------------------------------------------------
    // Option handling
    // -----------------------------------------------------------------------
    /// Encode an `Option<T>` at `src_offset`.
    ///
    /// Reads the option state via `is_some_fn` (0 = None, 1 = Some).
    /// - None: writes 0x00.
    /// - Some: writes 0x01, then jumps to `some_block` with `inner_ptr`.
    ///
    /// `get_value_fn` returns a pointer to the inner value when the option
    /// is Some; the encode block uses that pointer as its source base.
    EncodeOption {
        src_offset: usize,
        some_block: usize,
        /// vtable fn: `unsafe extern "C" fn(PtrConst) -> bool`
        is_some_fn: facet_core::OptionIsSomeFn,
        /// vtable fn: `unsafe extern "C" fn(PtrConst) -> PtrConst`
        get_value_fn: facet_core::OptionGetValueFn,
    },

    /// Encode an `Option<T>` using a calibrated in-memory layout — no vtable
    /// calls. The lowering probes `init_none` / `init_some(default)` (once over
    /// zero-filled memory, once over `0xFF`-filled memory) to discover which
    /// bytes rustc reliably writes and differ between the two variants. The
    /// JIT emits inline code that loads those tag bytes, XORs with their `None`
    /// values, ORs the results, and branches: zero → None, non-zero → Some.
    ///
    /// On None the emitted code writes postcard `0x00`; on Some it writes
    /// `0x01` then encodes the inner value at `src_offset + inner_offset`.
    EncodeOptionCalibrated {
        src_offset: usize,
        /// Byte offset from the option base to the inner `T` slot.
        inner_offset: usize,
        some_block: usize,
        /// Discriminator byte positions and their `None` values.
        tag_bytes: Box<[(usize, u8)]>,
    },

    /// Encode a `Result<T, E>` at `src_offset`.
    ///
    /// Writes postcard discriminant `0` for `Ok`, `1` for `Err`, then encodes
    /// the corresponding inner value from the pointer returned by the vtable.
    EncodeResult {
        shape: &'static Shape,
        src_offset: usize,
        ok_block: usize,
        err_block: usize,
        ok_shape: &'static Shape,
        err_shape: &'static Shape,
        is_ok_fn: facet_core::ResultIsOkFn,
        get_ok_fn: facet_core::ResultGetOkFn,
        get_err_fn: facet_core::ResultGetErrFn,
    },

    // -----------------------------------------------------------------------
    // Enum handling
    // -----------------------------------------------------------------------
    /// Write the enum variant's postcard index (its position in the variant
    /// list) as a varint. Emitted as the first op of each variant body so
    /// the index is correct regardless of any explicit Rust discriminant.
    WriteVariantIndex { index: u64 },

    /// Branch to the encode block for the active variant.
    ///
    /// Reads the tag at `src_offset` and dispatches to `variant_blocks[disc]`.
    /// `variant_blocks[i] = (discriminant_value, block_id)`.
    BranchOnEncode {
        src_offset: usize,
        tag_width: TagWidth,
        /// Parallel to variant_blocks: (disc_value, block_id).
        variant_blocks: Vec<(u64, usize)>,
    },

    // -----------------------------------------------------------------------
    // List / array handling
    // -----------------------------------------------------------------------
    /// Write the element count of a Vec-like container at `src_offset` as a
    /// varint, then iterate over elements using `body_block`.
    ///
    /// The body block is called once per element with base = element pointer.
    EncodeList {
        src_offset: usize,
        descriptor: OpaqueDescriptorId,
        body_block: usize,
        /// Byte stride between elements in the backing allocation.
        elem_size: usize,
    },

    /// Bulk-copy fast path for any calibrated container whose elements are
    /// stored in their wire-format-equivalent representation: write the
    /// varint length then a single memcpy of `len * elem_size` raw bytes into
    /// the output buffer. No per-element loop, no scalar dispatch.
    ///
    /// Eligible elements: `u8`/`i8` (1 byte raw, also covers `String`),
    /// `f32`/`f64` (postcard-spec'd as fixed LE; matches in-memory layout on
    /// LE hosts).
    WriteFixedList {
        src_offset: usize,
        descriptor: OpaqueDescriptorId,
        elem_size: usize,
    },

    /// Encode a fixed-size array at `src_offset`.
    ///
    /// Calls `body_block` exactly `count` times, advancing by `elem_size`.
    EncodeArray {
        src_offset: usize,
        count: usize,
        elem_size: usize,
        body_block: usize,
    },

    // -----------------------------------------------------------------------
    // Map handling
    // -----------------------------------------------------------------------
    /// Encode a map (`BTreeMap`/`HashMap`) at `src_offset`.
    ///
    /// Writes the varint entry count, then walks the map's `(key, value)` pairs
    /// through facet's iterator vtable. For each pair, `key_block` encodes the
    /// key with the source base set to the key pointer, and `value_block`
    /// encodes the value with the source base set to the value pointer.
    EncodeMap {
        src_offset: usize,
        /// facet `MapVTable::len` — entry count for the varint prefix.
        len_fn: facet_core::MapLenFn,
        /// facet `IterVTable::init_with_value` — creates the iterator state.
        iter_init_fn: facet_core::IterInitWithValueFn,
        /// facet `IterVTable::next` (Rust ABI) — yields the next `(key, value)`.
        iter_next_fn: facet_core::IterNextFn<(facet_core::PtrConst, facet_core::PtrConst)>,
        /// facet `IterVTable::dealloc` — frees the iterator state.
        iter_dealloc_fn: facet_core::IterDeallocFn,
        /// IR block encoding one key (source base = key pointer).
        key_block: usize,
        /// IR block encoding one value (source base = value pointer).
        value_block: usize,
    },

    // -----------------------------------------------------------------------
    // Control flow (mirrors decode IR)
    // -----------------------------------------------------------------------
    /// Unconditional jump to `block_id`.
    Jump { block_id: usize },

    /// Recursively encode the program's top shape from `src_offset` (relative
    /// to the current source pointer). JIT emits a direct self-call; emitted
    /// only when lowering hits a cycle back to the program's root (e.g.
    /// `enum Tree { Node(Box<Self>, Box<Self>), ... }`).
    CallSelf { src_offset: usize },

    /// End of the current block.
    Return,
}

/// A linear sequence of `EncodeOp` instructions.
#[derive(Debug, Clone, Default)]
pub struct EncodeBlock {
    pub ops: Vec<EncodeOp>,
}

/// A fully-lowered encode program for one root type.
///
/// Block 0 is always the entry point.
#[derive(Debug, Clone)]
pub struct EncodeProgram {
    pub blocks: Vec<EncodeBlock>,
    /// Size in bytes of the root value.
    pub root_size: usize,
    /// Alignment of the root value.
    pub root_align: usize,
    /// The shape this program encodes — set once at the start of lowering.
    /// Used to identify direct self-recursion and emit `CallSelf`.
    pub top_shape: Option<&'static Shape>,
    /// Shapes whose lowering is currently in progress on this stack — used
    /// for cycle detection. When `lower_encode_value` is called with a shape
    /// already in this set, recursion is detected.
    pub lowering_in_progress: HashSet<&'static Shape>,
}

impl EncodeProgram {
    fn new_block(&mut self) -> usize {
        let id = self.blocks.len();
        self.blocks.push(EncodeBlock::default());
        id
    }

    fn emit(&mut self, block: usize, op: EncodeOp) {
        self.blocks[block].ops.push(op);
    }
}

/// Error returned by the encode lowering pass.
#[derive(Debug)]
pub enum EncodeLowerError {
    /// The shape does not have a known sized layout.
    UnsizedShape,
    /// The enum representation is not stable (Rust or NPO repr).
    UnstableEnumRepr,
    /// The type is not supported by the JIT encode path.
    Unsupported(String),
}

impl std::fmt::Display for EncodeLowerError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::UnsizedShape => write!(f, "unsized shape"),
            Self::UnstableEnumRepr => write!(f, "unstable enum repr"),
            Self::Unsupported(s) => write!(f, "unsupported: {s}"),
        }
    }
}

/// Lower a `Shape` into an `EncodeProgram`.
///
/// No translation plan needed — encode always uses the local type definition.
/// `cal` is used to resolve Vec-family opaque descriptor handles.
pub fn lower_encode(
    shape: &'static Shape,
    cal: Option<&CalibrationRegistry>,
) -> Result<EncodeProgram, EncodeLowerError> {
    let layout = shape
        .layout
        .sized_layout()
        .map_err(|_| EncodeLowerError::UnsizedShape)?;
    let mut program = EncodeProgram {
        blocks: vec![EncodeBlock::default()],
        root_size: layout.size(),
        root_align: layout.align(),
        top_shape: Some(shape),
        lowering_in_progress: HashSet::new(),
    };
    lower_encode_value(shape, cal, &mut program, 0, 0)?;
    program.emit(0, EncodeOp::Return);
    debug_assert!(program.lowering_in_progress.is_empty());
    Ok(program)
}

fn lower_encode_value(
    shape: &'static Shape,
    cal: Option<&CalibrationRegistry>,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    // Cycle detection: if `shape` is already being lowered higher up the
    // stack we're at recursion (`Box<Self>` or similar). If it's recursion
    // back to the program's root, emit `CallSelf` — the JIT will recurse
    // through the function being compiled. Mutual recursion (in progress
    // but not the root) falls back to `SlowPath`, mirroring decode.
    if program.lowering_in_progress.contains(&shape) {
        if program.top_shape == Some(shape) {
            trace_cycle_emission(
                "encode",
                "CallSelf",
                shape,
                program.top_shape,
                &program.lowering_in_progress,
            );
            program.emit(block, EncodeOp::CallSelf { src_offset });
        } else {
            trace_cycle_emission(
                "encode",
                "SlowPath",
                shape,
                program.top_shape,
                &program.lowering_in_progress,
            );
            program.emit(block, EncodeOp::SlowPath { shape, src_offset });
        }
        return Ok(());
    }
    program.lowering_in_progress.insert(shape);
    let result = lower_encode_value_inner(shape, cal, program, block, src_offset);
    program.lowering_in_progress.remove(&shape);
    match result {
        Ok(()) => Ok(()),
        // Shape isn't statically lowerable (e.g. `facet_value::Value`, an
        // opaque dynamic, an unmodelled pointer/slice). The IR's `SlowPath`
        // op is the designed escape: at runtime the JIT-compiled stub calls
        // `vox_jit_encode_slow_path` which reflectively postcard-encodes
        // that one field. Strict callers opt out via `VOX_JIT_REQUIRE_PURE=1`,
        // which panics if any SlowPath remains in the program.
        Err(EncodeLowerError::Unsupported(_)) => {
            program.emit(block, EncodeOp::SlowPath { shape, src_offset });
            Ok(())
        }
        Err(e) => Err(e),
    }
}

fn lower_encode_value_inner(
    shape: &'static Shape,
    cal: Option<&CalibrationRegistry>,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    use facet_core::{Def, Type, UserType};

    // Transparent wrappers — pass through to inner shape
    if shape.is_transparent() {
        if let Type::User(UserType::Struct(st)) = shape.ty
            && let Some(inner_field) = st.fields.first()
        {
            return lower_encode_value(inner_field.shape(), cal, program, block, src_offset);
        }
        return Err(EncodeLowerError::Unsupported(format!(
            "transparent non-struct: {shape}"
        )));
    }

    if let Def::Result(result_def) = shape.def {
        return lower_encode_result(shape, result_def, cal, program, block, src_offset);
    }

    if shape.opaque_adapter.is_some() {
        program.emit(block, EncodeOp::WriteOpaque { shape, src_offset });
        return Ok(());
    }

    if shape.proxy.is_some() {
        program.emit(block, EncodeOp::WriteProxy { shape, src_offset });
        return Ok(());
    }

    // Scalars
    if let Some(scalar) = shape.scalar_type() {
        match scalar {
            facet_core::ScalarType::String => {
                if let Some(cal) = cal
                    && let Some(h) = cal.lookup_by_shape(shape)
                {
                    program.emit(
                        block,
                        EncodeOp::WriteFixedList {
                            src_offset,
                            descriptor: OpaqueDescriptorId(h.0),
                            elem_size: 1,
                        },
                    );
                    return Ok(());
                }
                program.emit(block, EncodeOp::WriteStringLike { shape, src_offset });
                return Ok(());
            }
            facet_core::ScalarType::Str | facet_core::ScalarType::CowStr => {
                program.emit(block, EncodeOp::WriteStringLike { shape, src_offset });
                return Ok(());
            }
            _ => {}
        }

        if let Some(prim) = wire_primitive_from_scalar(scalar) {
            program.emit(block, EncodeOp::WriteScalar { prim, src_offset });
            return Ok(());
        }
        return Err(EncodeLowerError::Unsupported(format!(
            "unknown scalar: {scalar:?}"
        )));
    }

    match shape.def {
        Def::Option(opt_def) => {
            return lower_encode_option(shape, opt_def, cal, program, block, src_offset);
        }
        Def::Array(arr_def) => {
            return lower_encode_array(arr_def, cal, program, block, src_offset);
        }
        Def::List(list_def) => {
            return lower_encode_list(shape, list_def, cal, program, block, src_offset);
        }
        Def::Pointer(ptr_def) => {
            return lower_encode_pointer(shape, ptr_def, cal, program, block, src_offset);
        }
        Def::Slice(slice_def) => {
            return lower_encode_slice(shape, slice_def, program, block, src_offset);
        }
        Def::Map(map_def) => {
            return lower_encode_map(shape, map_def, cal, program, block, src_offset);
        }
        Def::Set(_) => {
            // Set: same iterator strategy as Map applies once SetVTable's
            // iter_vtable is wired through — a fast-follow.
            return Err(EncodeLowerError::Unsupported(format!(
                "unsupported def: {shape}"
            )));
        }
        _ => {}
    }

    // User types: struct / enum
    match shape.ty {
        Type::User(UserType::Struct(st)) => {
            for field in st.fields {
                let field_offset = src_offset + field.offset;
                lower_encode_value(field.shape(), cal, program, block, field_offset)?;
            }
            Ok(())
        }
        Type::User(UserType::Enum(et)) => {
            lower_encode_enum(shape, et, cal, program, block, src_offset)
        }
        _ => Err(EncodeLowerError::Unsupported(format!(
            "unsupported type: {shape}"
        ))),
    }
}

fn lower_encode_pointer(
    shape: &'static Shape,
    ptr_def: facet_core::PointerDef,
    cal: Option<&CalibrationRegistry>,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    let Some(pointee_shape) = ptr_def.pointee() else {
        return Err(EncodeLowerError::Unsupported(
            "opaque pointer without pointee".into(),
        ));
    };

    if pointee_shape == <str as Facet<'static>>::SHAPE {
        program.emit(block, EncodeOp::WriteStringLike { shape, src_offset });
        return Ok(());
    }

    if let facet_core::Def::Slice(slice_def) = pointee_shape.def
        && slice_def.t().is_type::<u8>()
    {
        program.emit(block, EncodeOp::WriteBytesLike { shape, src_offset });
        return Ok(());
    }

    // Fast path for `Box<T>`/`SharedReference<T>` for sized T — the pointer's
    // in-memory layout is exactly a `*const T`, so codegen can emit a single
    // load instead of an indirect call to a vtable `borrow_fn`.
    let is_thin_pointer = matches!(
        ptr_def.known,
        Some(facet_core::KnownPointer::Box | facet_core::KnownPointer::SharedReference)
    );
    if is_thin_pointer {
        let body_block = program.new_block();
        program.emit(
            block,
            EncodeOp::DerefPointer {
                src_offset,
                body_block,
            },
        );
        lower_encode_value(pointee_shape, cal, program, body_block, 0)?;
        program.emit(body_block, EncodeOp::Return);
        return Ok(());
    }

    // General pointer-with-borrow_fn (Arc<T>, Rc<T>, Pin<P>, ...): emit
    // BorrowPointer to dereference, then lower the pointee inline at
    // src_offset = 0. lower_encode_value handles cycle detection — if the
    // pointee shape is the program's top, recursion becomes CallSelf.
    if let Some(borrow_fn) = ptr_def.vtable.borrow_fn {
        let body_block = program.new_block();
        program.emit(
            block,
            EncodeOp::BorrowPointer {
                src_offset,
                body_block,
                borrow_fn,
            },
        );
        lower_encode_value(pointee_shape, cal, program, body_block, 0)?;
        program.emit(body_block, EncodeOp::Return);
        return Ok(());
    }

    Err(EncodeLowerError::Unsupported(format!(
        "unsupported pointer: {pointee_shape}"
    )))
}

fn should_inline_loop_body(shape: &'static Shape) -> bool {
    if let Some(scalar) = shape.scalar_type() {
        let _ = scalar;
        return true;
    }

    match shape.def {
        facet_core::Def::Pointer(ptr_def) => {
            if let Some(pointee) = ptr_def.pointee() {
                return pointee.scalar_type() == Some(facet_core::ScalarType::Str)
                    || matches!(pointee.def, facet_core::Def::Slice(slice_def) if slice_def.t().is_type::<u8>());
            }
            false
        }
        facet_core::Def::Slice(slice_def) => slice_def.t().is_type::<u8>(),
        _ => false,
    }
}

fn lower_encode_slice(
    shape: &'static Shape,
    slice_def: facet_core::SliceDef,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    if !slice_def.t().is_type::<u8>() {
        return Err(EncodeLowerError::Unsupported(format!(
            "unsupported slice: {shape}"
        )));
    }

    program.emit(
        block,
        EncodeOp::WriteByteSlice {
            src_offset,
            len_fn: slice_def.vtable.len,
            as_ptr_fn: slice_def.vtable.as_ptr,
        },
    );
    Ok(())
}

fn lower_encode_option(
    shape: &'static Shape,
    opt_def: facet_core::OptionDef,
    cal: Option<&CalibrationRegistry>,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    let some_block = program.new_block();

    let is_niche_optimized = shape
        .layout
        .sized_layout()
        .ok()
        .zip(opt_def.t.layout.sized_layout().ok())
        .is_some_and(|(option_layout, inner_layout)| option_layout.size() == inner_layout.size());

    let fallback = match calibrate_option_layout(shape, opt_def) {
        // Same-size Option<T> layouts are niche-optimized. The decode path can
        // safely write calibrated None bytes and then overwrite the slot for
        // Some, but encode cannot classify arbitrary Some values from the None
        // bytes alone. Use facet's option vtable oracle for these layouts.
        Some(layout) if !is_niche_optimized && !layout.tag_bytes.is_empty() => {
            program.emit(
                block,
                EncodeOp::EncodeOptionCalibrated {
                    src_offset,
                    inner_offset: layout.inner_offset,
                    some_block,
                    tag_bytes: layout.tag_bytes,
                },
            );
            false
        }
        _ => true,
    };
    if fallback {
        program.emit(
            block,
            EncodeOp::EncodeOption {
                src_offset,
                some_block,
                is_some_fn: opt_def.vtable.is_some,
                get_value_fn: opt_def.vtable.get_value,
            },
        );
    }

    // Lower the inner value encode into some_block (base = inner_ptr, offset = 0).
    lower_encode_value(opt_def.t, cal, program, some_block, 0)?;
    program.emit(some_block, EncodeOp::Return);

    Ok(())
}

fn lower_encode_result(
    shape: &'static Shape,
    result_def: facet_core::ResultDef,
    cal: Option<&CalibrationRegistry>,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    let ok_block = program.new_block();
    let err_block = program.new_block();

    program.emit(
        block,
        EncodeOp::EncodeResult {
            shape,
            src_offset,
            ok_block,
            err_block,
            ok_shape: result_def.t,
            err_shape: result_def.e,
            is_ok_fn: result_def.vtable.is_ok,
            get_ok_fn: result_def.vtable.get_ok,
            get_err_fn: result_def.vtable.get_err,
        },
    );

    lower_encode_value(result_def.t, cal, program, ok_block, 0)?;
    program.emit(ok_block, EncodeOp::Return);

    lower_encode_value(result_def.e, cal, program, err_block, 0)?;
    program.emit(err_block, EncodeOp::Return);

    Ok(())
}

fn lower_encode_array(
    arr_def: facet_core::ArrayDef,
    cal: Option<&CalibrationRegistry>,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    let elem_shape = arr_def.t;
    let elem_layout = elem_shape
        .layout
        .sized_layout()
        .map_err(|_| EncodeLowerError::UnsizedShape)?;
    let elem_size = elem_layout.size();
    let body_block = program.new_block();

    program.emit(
        block,
        EncodeOp::EncodeArray {
            src_offset,
            count: arr_def.n,
            elem_size,
            body_block,
        },
    );

    if should_inline_loop_body(elem_shape) {
        lower_encode_value(elem_shape, cal, program, body_block, 0)?;
    } else {
        program.emit(
            body_block,
            EncodeOp::WriteShape {
                shape: elem_shape,
                src_offset: 0,
            },
        );
    }
    program.emit(body_block, EncodeOp::Return);

    Ok(())
}

/// Lower a map (`BTreeMap`/`HashMap`) to an `EncodeMap` op plus key/value body
/// blocks.
///
/// At runtime `EncodeMap` writes the varint entry count, then drives facet's
/// iterator vtable, encoding each key and value natively. Returns `Unsupported`
/// (so the caller falls back to `SlowPath`) for a map type that exposes no
/// iterator constructor.
fn lower_encode_map(
    shape: &'static Shape,
    map_def: facet_core::MapDef,
    cal: Option<&CalibrationRegistry>,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    let vt = map_def.vtable;
    let Some(iter_init_fn) = vt.iter_vtable.init_with_value else {
        return Err(EncodeLowerError::Unsupported(format!(
            "map type without an iterator constructor: {shape}"
        )));
    };

    let key_block = program.new_block();
    let value_block = program.new_block();
    program.emit(
        block,
        EncodeOp::EncodeMap {
            src_offset,
            len_fn: vt.len,
            iter_init_fn,
            iter_next_fn: vt.iter_vtable.next,
            iter_dealloc_fn: vt.iter_vtable.dealloc,
            key_block,
            value_block,
        },
    );

    // Key body: encode the key with source base = key pointer.
    lower_encode_value(map_def.k(), cal, program, key_block, 0)?;
    program.emit(key_block, EncodeOp::Return);

    // Value body: encode the value with source base = value pointer.
    lower_encode_value(map_def.v(), cal, program, value_block, 0)?;
    program.emit(value_block, EncodeOp::Return);

    Ok(())
}

fn lower_encode_list(
    shape: &'static Shape,
    list_def: facet_core::ListDef,
    cal: Option<&CalibrationRegistry>,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    let elem_shape = list_def.t;
    let elem_layout = elem_shape
        .layout
        .sized_layout()
        .map_err(|_| EncodeLowerError::UnsizedShape)?;
    let elem_size = elem_layout.size();

    // Look up calibration by structural shape identity (not pointer address).
    let descriptor = if let Some(cal) = cal
        && let Some(h) = cal.lookup_by_shape(shape)
    {
        OpaqueDescriptorId(h.0)
    } else {
        return Err(EncodeLowerError::Unsupported(format!(
            "Vec<T> without calibration: {shape}"
        )));
    };

    // Bulk-copy encode fast path: skip the per-element loop and emit
    // `varint(len) + memcpy(len * elem_size)` for `Vec<T>` where T's wire
    // format is bit-identical to its in-memory representation. Mirrors the
    // decode-side `ReadFixedVec` eligibility (u8/i8 always, f32/f64 on LE,
    // bool — Rust guarantees the byte is 0 or 1 already, so encode skips
    // validation).
    let is_byte_elem = elem_shape.is_type::<u8>() || elem_shape.is_type::<i8>();
    let is_bool_elem = elem_shape.is_type::<bool>();
    let is_float_elem_le = cfg!(target_endian = "little")
        && (elem_shape.is_type::<f32>() || elem_shape.is_type::<f64>());
    if is_byte_elem || is_bool_elem || is_float_elem_le {
        program.emit(
            block,
            EncodeOp::WriteFixedList {
                src_offset,
                descriptor,
                elem_size,
            },
        );
        return Ok(());
    }

    let body_block = program.new_block();

    program.emit(
        block,
        EncodeOp::EncodeList {
            src_offset,
            descriptor,
            body_block,
            elem_size,
        },
    );

    if should_inline_loop_body(elem_shape) {
        lower_encode_value(elem_shape, cal, program, body_block, 0)?;
    } else {
        program.emit(
            body_block,
            EncodeOp::WriteShape {
                shape: elem_shape,
                src_offset: 0,
            },
        );
    }
    program.emit(body_block, EncodeOp::Return);

    Ok(())
}

fn lower_encode_enum(
    _shape: &'static Shape,
    et: facet_core::EnumType,
    cal: Option<&CalibrationRegistry>,
    program: &mut EncodeProgram,
    block: usize,
    src_offset: usize,
) -> Result<(), EncodeLowerError> {
    let tag_width =
        tag_width_from_enum_repr(et.enum_repr).ok_or(EncodeLowerError::UnstableEnumRepr)?;

    // Each variant body first writes the variant's postcard index, then
    // encodes its fields. The dispatch block only branches — the index write
    // is deferred to the body so that explicit Rust discriminants (e.g.
    // `Inline = 1`) don't leak onto the wire. Postcard wants the variant's
    // position in the enum, not its in-memory tag byte.
    let mut variant_blocks: Vec<(u64, usize)> = Vec::new();
    for (i, variant) in et.variants.iter().enumerate() {
        let disc = variant.discriminant.map(|d| d as u64).unwrap_or(i as u64);
        let vblock = program.new_block();
        variant_blocks.push((disc, vblock));

        program.emit(vblock, EncodeOp::WriteVariantIndex { index: i as u64 });

        for field in variant.data.fields {
            let field_offset = src_offset + field.offset;
            lower_encode_value(field.shape(), cal, program, vblock, field_offset)?;
        }
        program.emit(vblock, EncodeOp::Return);
    }

    program.emit(
        block,
        EncodeOp::BranchOnEncode {
            src_offset,
            tag_width,
            variant_blocks,
        },
    );

    Ok(())
}