luna-jit 2.16.0

//! P12 — trace JIT data structures.
//!
//! Where `src/jit/mod.rs` is the *method* JIT (compiles one Proto's
//! body to cranelift IR), this module is the *trace* JIT: it records
//! one linear bytecode path through a hot back-edge, including
//! cross-call inlining, and compiles the recorded trace as a single
//! cranelift function with side-exit guards.
//!
//! The split lets the two co-exist: small leaf functions stay on the
//! method JIT (zero startup cost, no recording), while hot loops and
//! recursive functions move to the trace JIT once the per-Proto hot
//! counter passes [`TRACE_HOT_THRESHOLD`].
//!
//! ## Phase status
//!
//! - **S0** (commit 038c9ed): `Proto.trace_hot_count` field added.
//! - **S1** (this commit): trace data structures + dispatcher entry
//!   points. Counter wiring is feature-gated on
//!   `Vm.trace_enabled` (default `false`), so existing benchmarks
//!   are unaffected.
//! - **S2**: lower `TraceRecord` to cranelift IR, cache on
//!   `Proto.traces`.
//! - **S3**: `Vm::run` checks the trace cache at back-edge targets;
//!   side-exit returns continue in interpreter.
//! - **S4**: inline self-recursive `Op::Call` during recording.
//! - **S5**: escape analysis on `Op::NewTable` results.
//!

use luna_core::jit::send_compat::{TArc, TCellBool, TCellPtr, TCellU32, TRefLock};
use luna_core::runtime::Gc;
use luna_core::runtime::function::Proto;
use luna_core::vm::isa::{Inst, Op};

// v1.1 A1 Session C — pure data types live in
// `luna_core::jit::trace_types`. Re-export so existing
// `crate::jit_backend::trace::TraceRecord` paths within luna continue
// to resolve, and so the v1.0-compatible `luna_jit::jit::trace::*` surface
// (assembled in `luna_jit::lib::jit::trace`) sees both type defs and
// codegen entry points side-by-side.
pub use luna_core::jit::trace_types::*;

use cranelift::prelude::*;
use cranelift_codegen::ir::UserFuncName;
use cranelift_codegen::settings;
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_jit::{JITBuilder, JITModule};
use cranelift_module::{DataDescription, DataId, FuncId, Linkage, Module};

/// v1.3 Phase AOT Stage 7 sub-piece 2 — produce a `Value` of type
/// `I64` whose runtime contents are the live `Gc<LuaStr>` pointer for
/// `key_v`. Used by the four `iconst(I64, key.as_ptr() as i64)` sites
/// that feed `luna_jit_*_field` / `luna_jit_get_tab_up` helpers.
///
/// JIT path (`opts.aot == false`): emits `iconst(I64, key.as_ptr())`,
/// byte-for-byte identical to what the lowerer used to emit inline.
///
/// AOT path (`opts.aot == true`): declares two stably-named data
/// objects (deduped across calls via Cranelift's
/// `Module::declare_data` name interning):
///
/// - `__luna_aot_strkey_slot_<hex>` — 8-byte writable slot, defined
///   if not already defined this lower call, zero-initialised. Holds
///   the resolved `Gc<LuaStr>::as_ptr()` after the deploy-side
///   startup hook runs.
/// - `__luna_aot_strkey_bytes_<hex>` — read-only data carrying the
///   UTF-8 bytes the deploy `Vm` must intern.
///
/// `<hex>` is a 16-char SHA-256 prefix over the bytes (collision
/// risk: ~1 in 2^64 across the unique-string-key population of any
/// realistic Lua program). The bytes object is what the deploy-side
/// resolver walks; the slot is what the IR loads through.
///
/// Returns a `Value` holding the loaded `Gc<LuaStr>` pointer, suitable
/// for passing as the `key_arg` parameter to `luna_jit_set_field` /
/// `_get_field` / `_get_tab_up`.
fn emit_str_key_arg<M: Module>(
    module: &mut M,
    bcx: &mut FunctionBuilder<'_>,
    key_v: luna_core::runtime::Gc<luna_core::runtime::LuaStr>,
    aot: bool,
    defined_aot_data: &mut std::collections::HashSet<DataId>,
) -> Value {
    if !aot {
        return bcx.ins().iconst(types::I64, key_v.as_ptr() as i64);
    }
    let bytes = key_v.as_bytes();
    let hex = strkey_hex_label(bytes);

    // v1.3 Phase AOT Stage 7 sub-piece 3 — deploy-side resolver hook.
    //
    // In addition to the (slot, bytes) pair, we emit a third *index*
    // data symbol per unique key: 16 bytes of `[bytes_addr, slot_addr]`,
    // both populated by static-linker relocations. All index entries
    // land in a dedicated `__luna_aot_strkey_idx` section, so the
    // deploy resolver in `luna-runtime-helpers::luna_aot_run` can
    // walk the whole-program index between linker-provided start/end
    // bracket symbols (no per-binary enumeration scheme needed).
    //
    // Bracket symbol convention (declared on the resolver side):
    //   - ELF / lld: `__start_luna_strkey_idx` /
    //                `__stop_luna_strkey_idx` — auto-generated by the
    //                linker because the section name is a valid C
    //                identifier.
    //   - Mach-O:    `section$start$__DATA$luna_strkey_idx` /
    //                `section$end$__DATA$luna_strkey_idx` —
    //                Apple `ld`'s magic section-bracket synthesizer.
    //
    // Section name capped at 15 chars for the Mach-O 16-byte sectname
    // field (one slot for the implicit NUL when comparing).

    // 8-byte writable slot — IR loads through it.
    let slot_name = format!("__luna_aot_strkey_slot_{hex}");
    let slot_id = module
        .declare_data(&slot_name, Linkage::Export, true, false)
        .expect("declare_data slot");
    if defined_aot_data.insert(slot_id) {
        let mut desc = DataDescription::new();
        desc.define(Box::new([0u8; 8]));
        // `define_data` errors on redefinition — `insert` guards us.
        // Best-effort: ignore the error path (the only way `define`
        // fails here is duplicate-define, which our HashSet prevents).
        let _ = module.define_data(slot_id, &desc);
    }

    // Bytes manifest — deploy-side resolver reads via the index
    // section. Layout: little-endian u64 length, then the raw bytes
    // (no NUL). Read-only, non-TLS.
    let bytes_name = format!("__luna_aot_strkey_bytes_{hex}");
    let bytes_id = module
        .declare_data(&bytes_name, Linkage::Export, false, false)
        .expect("declare_data bytes");
    if defined_aot_data.insert(bytes_id) {
        let mut payload = Vec::with_capacity(8 + bytes.len());
        payload.extend_from_slice(&(bytes.len() as u64).to_le_bytes());
        payload.extend_from_slice(bytes);
        let mut desc = DataDescription::new();
        desc.define(payload.into_boxed_slice());
        let _ = module.define_data(bytes_id, &desc);
    }

    // Index entry — one per unique strkey, 16 bytes of
    // `[bytes_addr, slot_addr]` resolved by the static linker. Stable
    // name (`_idx_<hex>`) so multiple lower_trace_into calls within
    // one module don't emit duplicate entries (the resolver would
    // tolerate duplicates — re-intern is idempotent — but the link
    // bloat is wasted bytes).
    //
    // `Linkage::Local` because:
    //   - The symbol name is only an internal handle for the dedup
    //     guard; nothing outside this lowerer references it by name.
    //   - The deploy-side resolver walks the section, not by name.
    //   - Local keeps the global symbol table small.
    //
    // The dedicated `__luna_aot_strkey_idx` section lets the linker
    // auto-bracket the whole-program range; placing the entries in
    // the default rodata section would require an enumeration scheme
    // (per-`.o` registry table or `nm` parsing at startup).
    let idx_name = format!("__luna_aot_strkey_idx_{hex}");
    let idx_id = module
        .declare_data(&idx_name, Linkage::Local, false, false)
        .expect("declare_data idx");
    if defined_aot_data.insert(idx_id) {
        let mut desc = DataDescription::new();
        desc.define(Box::new([0u8; 16]));
        // Mach-O caps section names at 16 chars; `luna_strkey_idx`
        // is 15. ELF / COFF have no such limit but accept the same
        // short name — the deploy resolver brackets by the literal
        // section name on both platforms.
        // Use `__DATA` segment explicitly on Mach-O so this section
        // merges with the cmain shim's `__DATA,luna_strkey_idx`
        // placeholder. An empty segment string lands the section in
        // segment `""`, separate from `__DATA` — the deploy resolver's
        // `section$start$__DATA$luna_strkey_idx` would then bracket
        // only the placeholder, missing every real trace entry by 8
        // bytes (verified via `otool -lv` of an AOT binary). ELF / PE
        // ignore the segment arg (segment is Mach-O specific) so the
        // change is a no-op there.
        //
        // v1.3 Stage 7 polish 3 — Windows COFF host: PE section
        // headers are fixed 8 bytes, and `link.exe` / `lld-link` drop
        // the COFF long-name string table when producing the final PE.
        // Use the short name `.lt_skix` (8 bytes, matches the C
        // placeholder in `luna-aot::embed::write_aot_cmain_object_for`
        // Windows arm and the deploy-side
        // `windows_section::find_section` needle).
        let (idx_seg, idx_sect) = if cfg!(target_os = "windows") {
            ("", ".lt_skix")
        } else {
            ("__DATA", "luna_strkey_idx")
        };
        desc.set_segment_section(idx_seg, idx_sect);
        // 8-byte alignment for the two pointer relocations at offsets
        // 0 and 8. Mach-O `ld` hard-rejects unaligned pointer slots
        // ("pointer not aligned in `___luna_aot_strkey_idx_…`+0x8"),
        // which fires the first time a trace AOT-compiles a GetField/
        // SetField op (the strkey idx didn't exist in the trivial
        // smoke source).
        desc.set_align(8);
        // Two pointer-sized relocations: linker fills 0..8 with the
        // resolved address of `bytes_id` and 8..16 with `slot_id`.
        let bytes_gv = module.declare_data_in_data(bytes_id, &mut desc);
        let slot_gv = module.declare_data_in_data(slot_id, &mut desc);
        desc.write_data_addr(0, bytes_gv, 0);
        desc.write_data_addr(8, slot_gv, 0);
        let _ = module.define_data(idx_id, &desc);
    }

    // Emit the load through the slot.
    let slot_gv = module.declare_data_in_func(slot_id, bcx.func);
    let slot_addr = bcx.ins().symbol_value(types::I64, slot_gv);
    bcx.ins()
        .load(types::I64, MemFlags::trusted(), slot_addr, 0)
}

/// Stable 16-hex-char label for a string key. Pure FNV-1a 64-bit so we
/// don't pull `sha2` (luna-core 0-third-party-dep contract — this code
/// is in luna-jit, but the encoding stability matters for the
/// deploy-side resolver and a 64-bit FNV is collision-safe enough at
/// the trace-key scale).
fn strkey_hex_label(bytes: &[u8]) -> String {
    let mut h: u64 = 0xcbf29ce484222325;
    for &b in bytes {
        h ^= b as u64;
        h = h.wrapping_mul(0x100000001b3);
    }
    format!("{h:016x}")
}

/// v1.3 Phase AOT Stage 7 polish 6 — produce a `Value` of type `I64`
/// whose runtime contents are a pointer to the first
/// `FrameMaterializeInfo` of an inline cmp@d>0 side-exit's frame chain.
/// Used by the two `iconst(I64, Rc::as_ptr(&chain_rc) as ...)` sites
/// that feed `luna_jit_trace_materialize_frames(n, ptr)`.
///
/// JIT path (`opts.aot == false`): emits `iconst(I64, chain_first_ptr)`,
/// byte-for-byte identical to what the lowerer used to emit inline. The
/// caller still holds `chain_rc` alive via `per_exit_inline_vec`, so
/// the baked address is valid for the trace's mmap lifetime.
///
/// AOT path (`opts.aot == true`): declares three stably-named data
/// symbols (deduped by FNV-1a-64 of the packed chain bytes):
///
/// - `__luna_aot_inline_chain_slot_<hex>` — 8-byte writable u64,
///   zero-initialised. Deploy-side resolver writes the reconstructed
///   chain's first-element pointer here BEFORE any AOT trace dispatches.
/// - `__luna_aot_inline_chain_bytes_<hex>` — read-only, packed
///   `FrameMaterializeInfo` records: `[u64 count, (base_offset u32, pc
///   u32, nresults i32) * count]`. Same packing as the v3 wire-format
///   [`PerExitInlineEntry::chain_bytes`] field plus an 8-byte count
///   prefix so the resolver can size its reconstruction.
/// - `__luna_aot_inline_chain_idx_<hex>` — 16-byte `[bytes_ptr,
///   slot_ptr]` resolved by the static linker into the
///   `luna_inline_chain_idx` section so the deploy-side resolver can
///   walk every entry without per-binary enumeration.
///
/// IR emits `symbol_value(slot) + load(I64)`. The deploy resolver
/// (`luna-runtime-helpers::aot_inline_chain_resolver`) walks the idx
/// section, rebuilds a `Vec<FrameMaterializeInfo>` from the bytes
/// payload, materialises it as an `Rc<[...]>` whose ownership is
/// leaked into a process-lifetime store, then writes
/// `Rc::as_ptr() as *const FrameMaterializeInfo` into the slot. The
/// resolver runs BEFORE `aot_trace_registry::install_all` so the slot
/// is populated by the time `luna_jit_trace_materialize_frames` is
/// called from AOT mcode.
///
/// `<hex>` is FNV-1a-64 over the packed bytes (no count prefix in the
/// hash input — only the records). Collision risk: ~1 in 2^64 over
/// realistic trace populations.
fn emit_chain_ptr_arg<M: Module>(
    module: &mut M,
    bcx: &mut FunctionBuilder<'_>,
    chain: &[FrameMaterializeInfo],
    chain_rc_first_ptr: i64,
    aot: bool,
    defined_aot_data: &mut std::collections::HashSet<DataId>,
) -> Value {
    if !aot {
        return bcx.ins().iconst(types::I64, chain_rc_first_ptr);
    }
    // Pack the chain identically to the v3 wire format's chain_bytes:
    // three little-endian 32-bit fields per record. Drives both the
    // FNV hash (so identical chains share symbols across traces) and
    // the read-only bytes payload the resolver consumes.
    let mut packed: Vec<u8> = Vec::with_capacity(chain.len() * 12);
    for fm in chain {
        packed.extend_from_slice(&fm.base_offset.to_le_bytes());
        packed.extend_from_slice(&fm.pc.to_le_bytes());
        packed.extend_from_slice(&fm.nresults.to_le_bytes());
    }
    let hex = strkey_hex_label(&packed);

    // Writable 8-byte slot — IR loads through it. Zero at link time;
    // the deploy resolver populates with the rebuilt chain's pointer.
    let slot_name = format!("__luna_aot_inline_chain_slot_{hex}");
    let slot_id = module
        .declare_data(&slot_name, Linkage::Export, true, false)
        .expect("declare_data inline chain slot");
    if defined_aot_data.insert(slot_id) {
        let mut desc = DataDescription::new();
        desc.define(Box::new([0u8; 8]));
        let _ = module.define_data(slot_id, &desc);
    }

    // Bytes payload — `[u64 count, packed records]`, read-only. Count
    // prefix lets the resolver size its reconstruction without holding
    // an out-of-band length table.
    let bytes_name = format!("__luna_aot_inline_chain_bytes_{hex}");
    let bytes_id = module
        .declare_data(&bytes_name, Linkage::Export, false, false)
        .expect("declare_data inline chain bytes");
    if defined_aot_data.insert(bytes_id) {
        let mut payload = Vec::with_capacity(8 + packed.len());
        payload.extend_from_slice(&(chain.len() as u64).to_le_bytes());
        payload.extend_from_slice(&packed);
        let mut desc = DataDescription::new();
        desc.define(payload.into_boxed_slice());
        let _ = module.define_data(bytes_id, &desc);
    }

    // Index entry — one per unique chain, 16 bytes of
    // `[bytes_addr, slot_addr]` resolved by the static linker. Lives in
    // a dedicated `luna_inline_chain_idx` (Mach-O/ELF) / `.lt_chai`
    // (Windows COFF, 8-byte short-name limit) section so the deploy
    // resolver brackets the whole-program range.
    let idx_name = format!("__luna_aot_inline_chain_idx_{hex}");
    let idx_id = module
        .declare_data(&idx_name, Linkage::Local, false, false)
        .expect("declare_data inline chain idx");
    if defined_aot_data.insert(idx_id) {
        let mut desc = DataDescription::new();
        desc.define(Box::new([0u8; 16]));
        // Mach-O sectname max is 16 chars; `luna_inline_chain_idx`
        // is 21 — too long. Use the same 15-char `luna_inline_chnx`
        // shape (under the cap). Windows COFF short name 8-char cap →
        // `.lt_chai`. Both names must match the deploy resolver's
        // bracket / section-walker needles.
        let (idx_seg, idx_sect) = if cfg!(target_os = "windows") {
            ("", ".lt_chai")
        } else {
            ("__DATA", "luna_inline_chnx")
        };
        desc.set_segment_section(idx_seg, idx_sect);
        desc.set_align(8);
        let bytes_gv = module.declare_data_in_data(bytes_id, &mut desc);
        let slot_gv = module.declare_data_in_data(slot_id, &mut desc);
        desc.write_data_addr(0, bytes_gv, 0);
        desc.write_data_addr(8, slot_gv, 0);
        let _ = module.define_data(idx_id, &desc);
    }

    // Emit the load through the slot.
    let slot_gv = module.declare_data_in_func(slot_id, bcx.func);
    let slot_addr = bcx.ins().symbol_value(types::I64, slot_gv);
    bcx.ins()
        .load(types::I64, MemFlags::trusted(), slot_addr, 0)
}

/// Recognised single-arg libm math functions. Each entry maps the
/// Lua-side const-pool name (bytes) to the libm symbol cranelift
/// imports. Signature is uniform across the table: `(f64) -> f64`.
/// `math.log(x, base)` / `math.atan(y, x)` / `math.max(...)` use a
/// different bytecode window (B≠2) so the pattern matcher rejects
/// them.
const MATH_LIBM_FNS: &[(&[u8], &str)] = &[
    (b"sin", "sin"),
    (b"cos", "cos"),
    (b"tan", "tan"),
    (b"asin", "asin"),
    (b"acos", "acos"),
    (b"atan", "atan"),
    (b"exp", "exp"),
    (b"log", "log"),
    (b"sqrt", "sqrt"),
    (b"floor", "floor"),
    (b"ceil", "ceil"),
];

/// Kind of a recognised `math.<fn>(args...)` fold. `Libm1` is the
/// single-arg fold over libm (sin/cos/sqrt/floor/...) — emits a
/// libm call. `Min2` / `Max2` are 2-arg `math.min` / `math.max`
/// folds — emit Cranelift's native `fmin` / `fmax` (single mcode
/// insn on ARM64 and x86_64).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum FoldKind {
    Libm1,
    Min2,
    Max2,
}

/// Source of a Libm1 fold's argument register slot. Only `Move`
/// is accepted (existing v1.2 contract). The 2-arg `Min2 / Max2`
/// folds skip the [`FoldArgSrc`] decoding entirely — they emit at
/// the Call's position and read `R[A+1] / R[A+2]` directly, so
/// arg-prep ops can be arbitrarily-shaped (LoadI / Move / GetField
/// / Add / etc.) and execute normally before the Call's fold emit.
#[derive(Clone, Copy, Debug)]
enum FoldArgSrc {
    /// `Move R[A+k] := R[reg]`. Libm1 emit reads the variable at `reg`.
    Reg { reg: u32 },
}

/// A single recognised `math.<fn>(arg...)` fold.
///
/// Two shape kinds:
///
/// * **`Libm1`** — 4-op contiguous window: `GetTabUp + GetField +
///   Move + Call(B=2,C=2)`. The 4 indices `start_idx..start_idx+4`
///   are flagged in `folded_ops`. Emit fires at `start_idx` (the
///   `GetTabUp`) and produces one libm call.
///
/// * **`Min2 / Max2`** — *split-window* fold: `GetTabUp` at
///   `start_idx`, `GetField "min" / "max"` at `start_idx + 1`, and
///   `Call(B=3,C=2)` at `call_idx` (typically `start_idx + 4` for
///   `Move + Move` arg-prep, but can be `start_idx + 5` or more
///   when arg2 is computed by an in-place `GetField + Add` chain
///   like `bucket.tokens + refill`). Only `start_idx`,
///   `start_idx + 1`, and `call_idx` are flagged in `folded_ops`
///   — the arg-prep ops between them execute normally so the
///   Call's `R[A+1] / R[A+2]` arrive at their natural Lua-frame
///   slots. Emit fires at `call_idx` (the `Call`) and produces
///   `fmin / fmax` reading from `R[A+1] / R[A+2]`. The GetTabUp
///   and GetField emit positions are silent (their semantic result
///   — the resolved `math.min` function pointer — is discarded;
///   the trace knows the call target statically).
#[derive(Clone, Copy, Debug)]
struct TraceMathFold {
    start_idx: usize,
    /// Libm name ("sin", "cos", ...) for Libm1; "min" / "max"
    /// (diagnostic only) for Min2 / Max2.
    fn_name: &'static str,
    kind: FoldKind,
    /// Libm1 only: source of the single arg. `None` for Min2/Max2
    /// (their args live at R[A+1] / R[A+2] by Call ABI).
    arg_src: Option<FoldArgSrc>,
    /// Trace-index of the Op::Call this fold collapses. For
    /// `Libm1` it's `start_idx + 3`; for `Min2 / Max2` it's the
    /// recognised Call op's index. Used by `folded_ops` indexing
    /// and the emit-site `start_idx == i` lookup.
    call_idx: usize,
    /// `R[A]` of the Call — the destination of the fold's result.
    /// For `Libm1` this is also `start_idx`'s GetTabUp A; for
    /// `Min2 / Max2` it's identically `start_idx`'s GetTabUp A
    /// (and the Call's A).
    dst_reg: u32,
    /// `R[A+1]` of the Call — only meaningful for `Min2 / Max2`.
    arg1_reg: u32,
    /// `R[A+2]` of the Call — only meaningful for `Min2 / Max2`.
    arg2_reg: u32,
}

/// Maximum number of arg-prep ops the `Min2 / Max2` fold scans
/// between the `GetField "min"/"max"` and the closing `Call`.
/// Covers the common patterns (Move+Move = 2; LoadI+Move = 2;
/// LoadI + GetField + Add = 3; Move + GetField + Add + Add = 4) and
/// caps cost on adversarial trace shapes.
const MINMAX_FOLD_ARG_PREP_MAX: usize = 6;

/// Detect a `math.<fn>(arg...)` fold starting at recorded op index `i`.
/// Two arms recognised:
///   * `Libm1` — 4-op window `GetTabUp + GetField + Move + Call(B=2,C=2)`
///     mapped onto a libm fn in [`MATH_LIBM_FNS`].
///   * `Min2 / Max2` — split-window fold: `GetTabUp` at `i`, `GetField
///     "min"|"max"` at `i + 1`, then up to [`MINMAX_FOLD_ARG_PREP_MAX`]
///     arg-prep ops, then the closing `Call(B=3,C=2)`. The arg-prep
///     ops execute normally; only the GetTabUp/GetField/Call indices
///     are flagged in `folded_ops`. This lets `math.min(K, expr)`
///     fold even when `expr` is computed by an in-place `GetField +
///     Add` chain (the canonical Redis-Lua / BullMQ idiom).
///
/// Mirrors method JIT's `try_match_math_fold` but reads from
/// `record.ops[..]` instead of the Proto's code slice.
fn try_match_trace_math_fold(
    record: &TraceRecord,
    i: usize,
    head_proto: Gc<Proto>,
) -> Option<TraceMathFold> {
    // Common prefix (GetTabUp + GetField) needs at least 2 ops.
    if i + 1 >= record.ops.len() {
        return None;
    }
    // The first 2 ops must come from `head_proto` at depth 0.
    // Cross-Proto / inlined ops break the fold (would mis-resolve
    // the const-pool slots).
    for k in 0..=1 {
        if !std::ptr::eq(record.ops[i + k].proto.as_ptr(), head_proto.as_ptr()) {
            return None;
        }
        if record.ops[i + k].inline_depth != 0 {
            return None;
        }
    }
    let i0 = record.ops[i].inst;
    let i1 = record.ops[i + 1].inst;

    if !matches!(i0.op(), Op::GetTabUp) {
        return None;
    }
    if !matches!(i1.op(), Op::GetField) {
        return None;
    }

    let a = i0.a();
    // GetTabUp reads upvals[B] indexed by consts[C]. Pin B=0
    // (env upvalue) — frontend invariant.
    if i0.b() != 0 {
        return None;
    }
    let k_math = head_proto.consts.get(i0.c() as usize).copied()?;
    let luna_core::runtime::Value::Str(s) = k_math else {
        return None;
    };
    if s.as_bytes() != b"math" {
        return None;
    }

    // GetField R[A] = R[A].<key>. Same dest as source.
    if i1.a() != a || i1.b() != a {
        return None;
    }
    let k_fn = head_proto.consts.get(i1.c() as usize).copied()?;
    let luna_core::runtime::Value::Str(fname) = k_fn else {
        return None;
    };
    let fname_bytes = fname.as_bytes();

    // ── Libm1 arm (4-op window, B=2 C=2 call) ─────────────────
    if let Some(fn_name) = MATH_LIBM_FNS
        .iter()
        .find_map(|&(needle, name)| (needle == fname_bytes).then_some(name))
    {
        if i + 3 >= record.ops.len() {
            return None;
        }
        // Tail ops i+2, i+3 same head_proto + depth 0 constraint.
        for k in 2..=3 {
            if !std::ptr::eq(record.ops[i + k].proto.as_ptr(), head_proto.as_ptr()) {
                return None;
            }
            if record.ops[i + k].inline_depth != 0 {
                return None;
            }
        }
        let i2 = record.ops[i + 2].inst;
        let i3 = record.ops[i + 3].inst;
        if !matches!(i2.op(), Op::Move) {
            return None;
        }
        if !matches!(i3.op(), Op::Call) {
            return None;
        }
        // Move R[A+1] = R[arg_reg].
        if i2.a() != a + 1 {
            return None;
        }
        let arg_reg = i2.b();
        // Call R[A], B=2 (1 arg), C=2 (1 result).
        if i3.a() != a || i3.b() != 2 || i3.c() != 2 {
            return None;
        }
        return Some(TraceMathFold {
            start_idx: i,
            fn_name,
            kind: FoldKind::Libm1,
            arg_src: Some(FoldArgSrc::Reg { reg: arg_reg }),
            call_idx: i + 3,
            dst_reg: a,
            arg1_reg: 0,
            arg2_reg: 0,
        });
    }

    // ── Min2 / Max2 arm (split-window, B=3 C=2 call) ───────────
    //
    // GetTabUp at `i`, GetField at `i+1`, then scan forward up to
    // MINMAX_FOLD_ARG_PREP_MAX ops looking for the matching
    // `Op::Call A B=3 C=2`. All scanned ops must come from
    // `head_proto` at depth 0. We do NOT constrain the shape of
    // the arg-prep ops — they execute normally and leave the call
    // args at `R[a+1]` / `R[a+2]` by the standard Lua Call ABI.
    let kind = match fname_bytes {
        b"min" => FoldKind::Min2,
        b"max" => FoldKind::Max2,
        _ => return None,
    };
    let mut call_idx: Option<usize> = None;
    let search_limit = (i + 2 + MINMAX_FOLD_ARG_PREP_MAX + 1).min(record.ops.len());
    for j in (i + 2)..search_limit {
        let rop_j = &record.ops[j];
        if !std::ptr::eq(rop_j.proto.as_ptr(), head_proto.as_ptr()) || rop_j.inline_depth != 0 {
            return None;
        }
        let inst_j = rop_j.inst;
        if matches!(inst_j.op(), Op::Call) {
            // Call R[A], B=3 (2 args), C=2 (1 result), A matches
            // the GetTabUp dest (the fn lives in R[A]).
            if inst_j.a() != a || inst_j.b() != 3 || inst_j.c() != 2 {
                return None;
            }
            call_idx = Some(j);
            break;
        }
        // Arg-prep ops between GetField and Call must not overwrite
        // the fn slot R[A] — the Call expects R[A] = the resolved
        // math.<fn>. (In practice the parser allocates A+1/A+2 for
        // args, leaving R[A] untouched, but we double-check.)
        if inst_j.a() == a {
            return None;
        }
    }
    let call_idx = call_idx?;
    let diag_name = if matches!(kind, FoldKind::Min2) {
        "min"
    } else {
        "max"
    };
    Some(TraceMathFold {
        start_idx: i,
        fn_name: diag_name,
        kind,
        arg_src: None,
        call_idx,
        dst_reg: a,
        arg1_reg: a + 1,
        arg2_reg: a + 2,
    })
}

/// Single-op classifier — the per-op decision logic used by the
/// look-ahead walker [`infer_getx_exit_lookahead`].
///
/// The helpers returning a GetX value (`Op::GetI` / `Op::GetTable` /
/// `Op::GetField` / `Op::GetTabUp`) hand back the table cell's raw
/// 8-byte payload — Int, Table, Float, or anything else. A static
/// prediction of the payload's tag needs context from the use site.
///
/// - Arithmetic / numeric cmp operand → must be Int.
/// - Table-base operand (Get / Set / Len) → must be Table.
/// - Anything else (`Move`, `Eq` (bitwise-ok), trace tail, literal
///   materialisation) → unknown; return `None`. The walker treats
///   `LoadI / LoadF / LoadK` as transparent and walks past them.
///
/// `Op::Eq` is *not* a tag indicator: `icmp eq` of i64 payloads
/// catches both `Int == 0` and `Table == nil` correctly, so the
/// cmp itself doesn't pin the result's tag.
fn infer_getx_exit_inst(getx_a: u32, next: Inst) -> Option<ExitTag> {
    let na = next.a();
    let nb = next.b();
    let nc = next.c();
    match next.op() {
        // Arith: A := B op C. The result's reg is Int; if a
        // GetX output is consumed here it's an Int operand.
        Op::Add | Op::Sub | Op::Mul => {
            if nb == getx_a || nc == getx_a {
                Some(ExitTag::Int)
            } else {
                None
            }
        }
        // Lt / Le compare ordering — operand must be numeric.
        Op::Lt | Op::Le => {
            if na == getx_a || nb == getx_a {
                Some(ExitTag::Int)
            } else {
                None
            }
        }
        // Table-base reads: A := B[*]. The B operand must be a
        // table; if GetX's output feeds it, we know it's a Table.
        Op::GetI | Op::GetTable | Op::GetField => {
            if nb == getx_a {
                Some(ExitTag::Table)
            } else {
                None
            }
        }
        // Table-base writes: A[*] := *. The A operand must be a
        // table.
        Op::SetI | Op::SetTable | Op::SetList | Op::SetField => {
            if na == getx_a {
                Some(ExitTag::Table)
            } else {
                None
            }
        }
        // `#R[B]` requires R[B] to be a table.
        Op::Len => {
            if nb == getx_a {
                Some(ExitTag::Table)
            } else {
                None
            }
        }
        // `Op::Eq` does bitwise equality on i64 payloads — works
        // for both Int and Table comparisons, so it tells us
        // nothing about the operand's tag.
        _ => None,
    }
}

/// Look-ahead variant of [`infer_getx_exit`]: walks the recorded ops
/// after the GetX, skipping ops that are *transparent* (provably
/// don't read `R[getx_a]` and don't overwrite it). Returns as soon
/// as the first non-transparent op classifies the use, or `None` if
/// nothing in the trail consumes the slot or the slot is overwritten
/// first.
///
/// The transparent set is intentionally tight: only `LoadI`,
/// `LoadF`, `LoadK` (literal materialisation into a register slot ≠
/// getx_a) qualify. These ops never read any register, so they can
/// never consume `R[getx_a]`; if their `A` ≠ `getx_a` they also
/// don't overwrite it. Anything else stops the walk — either we
/// classify (Add / Sub / Mul / Lt / Le / Get* / Set* / Len) or
/// we conservatively return `None`.
///
/// This unblocks the common Lua codegen pattern
///
/// ```text
///   GetField R[a], R[base], "k"   ; read
///   LoadI    R[a+1], <literal>    ; materialise const operand
///   Le       R[?],  R[a], R[a+1]  ; compare R[a] vs literal
/// ```
///
/// where the 1-op-ahead path saw `LoadI` next and bailed.
fn infer_getx_exit_lookahead(getx_a: u32, ops_after: &[RecordedOp]) -> Option<ExitTag> {
    // Bound the walk — analysis cost cap on faulty trace shapes.
    // Most use sites are within 1-2 ops; 4 is generous.
    const MAX_LOOKAHEAD: usize = 4;
    for rop in ops_after.iter().take(MAX_LOOKAHEAD) {
        let inst = rop.inst;
        let op = inst.op();
        let a = inst.a();
        // First, a per-op classification attempt — if the use site
        // is right here, that wins.
        if let Some(tag) = infer_getx_exit_inst(getx_a, inst) {
            return Some(tag);
        }
        // Transparent: `LoadI / LoadF / LoadK` materialise a
        // literal into `R[A]`. They read no registers. If `A` is
        // not our slot they don't affect it; walk past. This
        // covers the canonical Lua codegen pattern
        // `GetField R[a]; LoadI R[a+1], K; Le R[?], R[a], R[a+1]`
        // where the 1-op-ahead path would have stopped at LoadI.
        if matches!(op, Op::LoadI | Op::LoadF | Op::LoadK) {
            if a == getx_a {
                // LoadX overwrites our slot before any use — give up.
                return None;
            }
            continue;
        }
        // Any other op stops the walk: either the per-op classifier
        // already pinned a tag (handled above) or we conservatively
        // bail to avoid a wrong static prediction.
        return None;
    }
    None
}

/// P12-S4-step2c — forward-look exit-tag inference for `Op::GetUpval`.
/// Walks the recorded ops following the GetUpval until either:
/// - an `Op::Call` with `A == getupval_a` is found → the upval is
///   that Call's function target → `Some(ExitTag::Closure)`.
/// - an op that writes `R[getupval_a]` is found before the Call →
///   the upval is overwritten in this slot → `None`.
/// - the walk runs out → `None`.
fn infer_upval_exit(getupval_a: u32, ops_after: &[RecordedOp]) -> Option<ExitTag> {
    for rop in ops_after {
        let next = rop.inst;
        if next.op() == Op::Call && next.a() == getupval_a {
            return Some(ExitTag::Closure);
        }
        // Conservative writes-A detection: ops that don't write A
        // are control / cmp / store ops. Everything else writes A.
        let writes_a = !matches!(
            next.op(),
            Op::Lt
                | Op::Le
                | Op::Eq
                | Op::EqK
                | Op::Jmp
                | Op::SetI
                | Op::SetTable
                | Op::SetList
                | Op::Return0
                | Op::Return1
                | Op::Return
        );
        if writes_a && next.a() == getupval_a {
            return None;
        }
    }
    None
}

// `pub enum ExitTag` moved to `trace_types.rs`; re-exported via
// `pub use super::trace_types::*;` at the file head.

// P13-S13-F — `ExitTag::MoveFrom(u8)` was deprecated by S2's
// kind-propagation rewrite: `current_kinds` now tracks Move
// sources at the Move op, so the dispatcher no longer needs a
// deferred entry-tag lookup. The variant was last produced by
// pre-S2 emit; modern emit + dispatcher don't reference it.
// Dropping the variant lets ExitTag fit in a single byte (no
// payload), which Rc<[ExitTag]> in CompiledTrace exit_tags +
// per_exit_tags benefits from at the cache-line level.

/// Per-register *current* kind tracked during the lowerer's forward
/// sweep. Initial values come from the trace's `entry_tags`
/// snapshot, then arith / Move / GetX / NewTable writers refine
/// them. Used by arith / cmp emit to pick the right IR (iadd vs
/// fadd; icmp vs fcmp).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum RegKind {
    Unset,
    Int,
    Float,
    Table,
    /// P12-S4-step2c — Lua closure pointer (raw payload is a
    /// `Gc<LuaClosure>` ptr). Produced today only by `Op::GetUpval`
    /// when `infer_upval_exit` pins the use site as Op::Call's
    /// target.
    Closure,
    Nil,
    /// P12-S12-C v2 — interned string pointer (raw payload is a
    /// `Gc<LuaStr>` ptr). Produced by an entry slot tagged STR
    /// (via `from_entry_tag`), `LoadK` of a Str constant, or
    /// `Op::Move` propagation from a Str slot. Lets `Op::Concat`
    /// emit's operand spill pick `raw::STR` instead of going
    /// through the tag-preserving `update_raw` helper which can't
    /// handle a stale vm.stack tag.
    Str,
}

impl RegKind {
    fn from_entry_tag(tag: u8) -> Option<Self> {
        match tag {
            luna_core::runtime::value::raw::INT => Some(RegKind::Int),
            luna_core::runtime::value::raw::FLOAT => Some(RegKind::Float),
            luna_core::runtime::value::raw::TABLE => Some(RegKind::Table),
            luna_core::runtime::value::raw::CLOSURE => Some(RegKind::Closure),
            luna_core::runtime::value::raw::NIL => Some(RegKind::Nil),
            luna_core::runtime::value::raw::STR => Some(RegKind::Str),
            _ => None,
        }
    }
}

/// P12-S4-step3a — per-op register window offset.
///
/// For a recorded trace with inline self-recursive `Op::Call`s, each
/// `RecordedOp` at depth `d` views its `R[k]` as the trace's
/// `reg_state_buf[offset_per_op[i] + k]`. The offset accumulates
/// across self-recursive calls:
/// - depth 0 ops: offset = 0
/// - When `Op::Call A B C` (self-recursive) at depth d is followed
///   by an op at depth d+1: that callee's offset = `offset[d] + A + 1`.
///   (Lua call ABI: callee's `R[0]` lives at caller's `R[A+1]`, since
///   `R[A]` is the function value being called.)
/// - On `Op::Return*` at depth d>0: depth drops to d-1, offset
///   reverts to the saved `offset[d-1]`.
///
/// Used by S4-step3b's body emit to address registers across inlined
/// frames. The companion `enclosing_call_a` Vec gives the matching
/// caller `Op::Call`'s A field for any depth>0 op (None at depth 0),
/// which step 3b's `Op::Return*` emit consumes to compute the
/// caller's destination slot for return-value copy-back.
/// P13-S13-A — pure-function depth invariant verifier.
///
/// Returns `true` iff the recorded op sequence's inline-depth
/// trail is well-formed for `compute_op_offsets` to consume:
///
/// 1. The first op (if any) is at depth 0.
/// 2. Any depth bump (`d_curr > d_prev`) is exactly `d_prev + 1`
///    AND the previous op is an `Op::Call` (the only legitimate
///    frame-push trigger in Lua bytecode under the recorder's
///    self-rec inline contract).
/// 3. Depth never exceeds `MAX_INLINE_DEPTH` (the lowerer's
///    window-size cap).
///
/// Depth descents (`d_curr < d_prev`) are unconstrained — the
/// interpreter can unwind many frames via consecutive
/// `Op::Return*` / tail-call exits, and the offset stack `pop()`
/// loop in `compute_op_offsets` handles arbitrary descent.
///
/// The function operates on `(depth, is_call)` tuples instead of
/// the heavier `RecordedOp` so the lib unit tests can construct
/// synthetic input without a real `Gc<Proto>`.
pub(crate) fn verify_depth_invariant(items: &[(u8, bool)]) -> bool {
    if items.is_empty() {
        return true;
    }
    if items[0].0 != 0 {
        return false;
    }
    let mut prev_depth = items[0].0;
    let mut prev_was_call = items[0].1;
    for &(d, is_call) in &items[1..] {
        if d > prev_depth {
            if d != prev_depth + 1 {
                return false;
            }
            if !prev_was_call {
                return false;
            }
        }
        if d > MAX_INLINE_DEPTH {
            return false;
        }
        prev_depth = d;
        prev_was_call = is_call;
    }
    true
}

fn compute_op_offsets(record: &TraceRecord) -> (Vec<u32>, Vec<Option<u8>>) {
    let n = record.ops.len();
    let mut offsets = Vec::with_capacity(n);
    let mut enclosing_call_a = Vec::with_capacity(n);
    // `offset_stack[d]` = the register-window offset for depth d.
    // `call_a_stack[d]` = the Op::Call A field that entered depth d.
    //   `call_a_stack[0]` is unused (depth 0 has no enclosing call).
    let mut offset_stack: Vec<u32> = vec![0u32];
    let mut call_a_stack: Vec<u8> = vec![0u8];
    for (i, rop) in record.ops.iter().enumerate() {
        let d = rop.inline_depth as usize;
        if d >= offset_stack.len() {
            // Depth increased — the previous op must be Op::Call
            // (recorder invariant for self-recursive entry).
            debug_assert!(i > 0, "first recorded op cannot be at depth > 0");
            debug_assert!(
                d == offset_stack.len(),
                "depth jumped more than 1 in a single transition"
            );
            let caller_idx = i - 1;
            let caller = &record.ops[caller_idx];
            debug_assert!(
                matches!(caller.inst.op(), Op::Call),
                "depth bump must follow Op::Call"
            );
            let caller_offset = offset_stack[offset_stack.len() - 1];
            let caller_a = caller.inst.a();
            let new_offset = caller_offset + caller_a + 1;
            offset_stack.push(new_offset);
            // Lua register indices fit in u8 by VM design; this
            // cast is lossless for any valid bytecode.
            call_a_stack.push(caller_a as u8);
        } else {
            // Depth decreased (or stayed equal). Pop down to d.
            while offset_stack.len() > d + 1 {
                offset_stack.pop();
                call_a_stack.pop();
            }
        }
        offsets.push(offset_stack[d]);
        enclosing_call_a.push(if d == 0 { None } else { Some(call_a_stack[d]) });
    }
    (offsets, enclosing_call_a)
}

// `pub enum TagResKind` + `pub(crate) fn classify_exit_tags` moved to
// `trace_types.rs`; re-exported via `pub use super::trace_types::*;`.

fn kinds_to_exit_tags(kinds: &[RegKind]) -> Vec<ExitTag> {
    kinds
        .iter()
        .map(|k| match k {
            RegKind::Unset => ExitTag::Untouched,
            RegKind::Int => ExitTag::Int,
            RegKind::Float => ExitTag::Float,
            RegKind::Table => ExitTag::Table,
            RegKind::Closure => ExitTag::Closure,
            RegKind::Nil => ExitTag::Nil,
            RegKind::Str => ExitTag::Str,
        })
        .collect()
}

/// P12-S5-A — escape state per NewTable site recorded in a trace.
/// Today purely diagnostic; S5-B will read this to drive scalar
/// replacement of the array part on Sinkable sites.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EscapeState {
    /// No observed use forces heap residency; emit may sink (S5-B+).
    Sinkable,
    /// A use forced materialisation — call argument, return value,
    /// stored into another sinkable table, observed at a side-exit.
    Escaped,
}

/// P12-S5-A — one NewTable site found in a recorded trace, tagged
/// with the slot it lives in and the final [`EscapeState`].
#[derive(Debug, Clone)]
pub struct AllocSite {
    /// Index into `record.ops` of the NewTable op.
    pub op_idx: usize,
    /// Bytecode PC of the NewTable (in the site's enclosing Proto).
    pub pc: u32,
    /// Destination register `R[A]` = newly-allocated table.
    pub a: u32,
    /// Inline depth at the time of the NewTable (0 = trace head).
    pub inline_depth: u8,
    /// Array capacity decoded from NewTable.B (S5-A handles the
    /// `B > 0, C = 0` form for array sunk; S11-B-v1 admits `B = 0`
    /// for hash-only sites — array_cap = 0 in that case).
    pub array_cap: u32,
    /// P12-S11-B-v1 — unique string-key const indices touched by
    /// `Op::SetField` / `Op::GetField` on this site (in scan order).
    /// `virt_vars` indices [array_cap .. array_cap + hash_keys.len())
    /// hold the hash slots; SetFieldSunkWrite / GetFieldSunkRead
    /// look up the key's position in this vec to pick the slot.
    pub hash_keys: Vec<u32>,
    /// Final escape-state classification after the sweep.
    pub state: EscapeState,
}

/// P12-S5-B — per-op action recorded by the escape sweep when an op
/// reads or writes through a NewTable site. Emit consults this to
/// take the sunk path (no helper call, virtual `Variable`s) instead
/// of the heap-alloc helper path. Only set for ops whose sunk path
/// is implemented today (S5-B v1: `NewTable`, `SetList`, `GetI`).
/// Sweep marks the site Escaped on any unsupported op (SetI /
/// SetTable / GetTable / Len / Move / Call arg / Return) so emit
/// can rely on "Sinkable → all-ops sunk".
#[derive(Debug, Clone, Copy)]
pub enum OpAction {
    /// `Op::NewTable` allocating site_idx. Emit skips the
    /// `luna_jit_new_table` helper; the site lives as virtual slots.
    NewTableSite {
        /// Allocation site index in [`EscapeAnalysis::sites`].
        site_idx: u32,
    },
    /// `Op::SetList` writing into a sunk site's array. Emit
    /// `def_var`s the source registers into virtual slots.
    SetListWrite {
        /// Allocation site index.
        site_idx: u32,
    },
    /// `Op::GetI` reading from a sunk site at a 1-based key. Emit
    /// `def_var`s the corresponding virtual slot into the dst reg.
    GetIRead {
        /// Allocation site index.
        site_idx: u32,
        /// 1-based array key being read.
        key: u32,
    },
    /// P12-S8-B — `Op::SetI` writing into a sunk site's slot at a
    /// 1-based key. Emit `def_var`s the value register into the
    /// matching virt slot Variable and updates `virt_kinds` so the
    /// next `GetI` reads the right RegKind. The value source is
    /// the runtime register `R[C]`.
    SetISunkWrite {
        /// Allocation site index.
        site_idx: u32,
        /// 1-based array key being written.
        key: u32,
    },
    /// P12-S8-C — `Op::SetTable` writing into a sunk site's slot
    /// at a 1-based key const-folded from a backward scan of the
    /// trace (LoadI → Move chain → key). Same emit shape as
    /// SetISunkWrite; the key field is the resolved int.
    SetTableSunkWrite {
        /// Allocation site index.
        site_idx: u32,
        /// Const-folded 1-based array key.
        key: u32,
    },
    /// P12-S11-B-v1 — `Op::SetField` writing into a sunk site's
    /// hash slot. `hash_slot` is the position of the key's const
    /// index in `AllocSite.hash_keys`. virt_vars index is
    /// `array_cap + hash_slot`.
    SetFieldSunkWrite {
        /// Allocation site index.
        site_idx: u32,
        /// Position in [`AllocSite::hash_keys`] of the field key.
        hash_slot: u32,
    },
    /// P12-S11-B-v1 — `Op::GetField` reading from a sunk site's
    /// hash slot. Same indexing as SetFieldSunkWrite.
    GetFieldSunkRead {
        /// Allocation site index.
        site_idx: u32,
        /// Position in [`AllocSite::hash_keys`] of the field key.
        hash_slot: u32,
    },
}

/// P14-S14-B v0 — buffer state for a candidate `Op::Concat`
/// accumulator. Mirrors S5's [`EscapeState`] semantics.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BufferState {
    /// No observed use forces materialisation of the buffered slot;
    /// emit may take the buffered path (per-trace `Vec<u8>` append).
    /// Held back for v1+ — the v0 detector never emits this.
    Bufferable,
    /// A use forced materialisation — accumulator passed to a Call,
    /// Move'd elsewhere, observed at a side-exit with Str exit_tag,
    /// etc. Emit falls back to the existing `luna_jit_op_concat`
    /// helper path.
    NonBuffered,
}

/// P14-S14-B v0 — one `Op::Concat A B=2` candidate found by
/// `detect_accumulators` where `A` (the destination) is the same
/// register as the first operand AND survives across the trace's
/// back-edge. v0 ships the struct + an empty detection pass (no
/// candidates produced yet) so v1+ can extend without API churn.
#[derive(Debug, Clone)]
pub struct AccumSite {
    /// Index into `record.ops` of the `Op::Concat` op.
    pub op_idx: usize,
    /// Bytecode PC of the `Op::Concat`.
    pub pc: u32,
    /// The accumulator slot (= `Op::Concat.A`). Also reads as first
    /// operand and writes the result.
    pub accum_slot: u32,
    /// The piece slot (= `Op::Concat.A + 1` since `B = 2`).
    pub piece_slot: u32,
    /// Inline depth — v1 restricts to 0.
    pub inline_depth: u8,
    /// Final buffer-state classification after escape-style sweep.
    pub state: BufferState,
}

/// P12-S5-A — result of the post-recording, pre-emit escape sweep
/// run by [`try_compile_trace_with_options`]. S5-B emit reads
/// `sites` (for the Sinkable list) and `op_actions` (per-op
/// dispatch hint). S5-C consumes `live_at_op` at every cmp
/// side-exit emit point to materialise the right virt slots.
#[derive(Debug, Default)]
pub struct EscapeAnalysis {
    /// One [`AllocSite`] per `Op::NewTable` in the trace.
    pub sites: Vec<AllocSite>,
    /// Per-op action, length = `effective_end`. Indexed by op index
    /// in `record.ops`. `None` for ops the sweep didn't tag.
    pub op_actions: Vec<Option<OpAction>>,
    /// Per-op snapshot of bound site indices (sites whose binding
    /// is live BEFORE this op processes), length = `effective_end`.
    /// S5-C reads this at cmp emit sites to know which sites'
    /// virt slots must materialise into a heap `Gc<Table>` on
    /// side-exit. Sites that end up Escaped after the sweep are
    /// still in the snapshot but emit gates on the final state.
    pub live_at_op: Vec<Vec<u32>>,
    /// P14-S14-B v0 — accumulator sites identified in this trace.
    /// Empty in v0 (`detect_accumulators` is a stub); v1+ will
    /// populate.
    pub accum_sites: Vec<AccumSite>,
    /// P14-S14-B v0 — per-op snapshot of bound accumulator-site
    /// indices, parallel to `live_at_op`. Length = `effective_end`
    /// once populated; empty `Vec<Vec<u32>>` in v0.
    pub accum_live_at_op: Vec<Vec<u32>>,
}

impl EscapeAnalysis {
    /// Count of sites whose final state is [`EscapeState::Sinkable`].
    pub fn sinkable_count(&self) -> u32 {
        self.sites
            .iter()
            .filter(|s| s.state == EscapeState::Sinkable)
            .count() as u32
    }
}

/// P12-S5-C — map a [`RegKind`] to its matching
/// [`luna_core::runtime::value::raw`] tag byte for the materialise
/// helper's per-slot kind array. `Unset` slots become `NIL` so the
/// helper writes Nil into the corresponding array index — matches
/// Lua's "table created with array part, slot unwritten" semantics.
fn kind_to_raw_tag(k: RegKind) -> u8 {
    use luna_core::runtime::value::raw;
    match k {
        RegKind::Int => raw::INT,
        RegKind::Float => raw::FLOAT,
        RegKind::Table => raw::TABLE,
        RegKind::Closure => raw::CLOSURE,
        RegKind::Str => raw::STR,
        RegKind::Unset | RegKind::Nil => raw::NIL,
    }
}

/// P12-S5-C — at a cmp side-exit emit point, materialise every
/// live Sinkable site at `inline_depth = 0`. For each site:
/// stack-allocate two parallel buffers (`cap × i64` raws + `cap × u8`
/// kind tags), fill from virt slot Variables + `virt_kinds`, call
/// the materialise helper, and `def_var` the returned heap table
/// bits into `regs_full[site.a]` so the subsequent `store_back`
/// lands the heap pointer in `reg_state[site.a]`. Returns the
/// per-exit-tags snapshot (with materialised slots overridden to
/// `RegKind::Table`) plus the number of sites materialised at this
/// emit point.
///
/// `inline_depth > 0` sites are demoted in pre-emit (the `has_inline_cmp`
/// gate), so this function only walks depth-0 sites. Inline sinking
/// is a follow-up (would extend `regs_full[off + site.a]` indexing
/// for the inlined frame's window).
/// P12-S5-C / S10-A / S10-B — at a cmp side-exit emit point,
/// materialise every live Sinkable site (depth=0 AND depth>0) into
/// the heap and update `kinds_snapshot` so the dispatcher's restore
/// path repacks `RegKind::Table` for each materialised slot.
///
/// `kinds_snapshot` is updated in-place for each materialised
/// site's slot — caller passes either a `max_stack`-sized snapshot
/// (depth=0 cmp arm; bounds check skips depth>0 sites by index)
/// or a window-sized snapshot (depth>0 cmp arm via S10-B; depth>0
/// sites land inside the window).
///
/// Returns the number of sites materialised at this emit point.
fn emit_materialize_live_sunk<M: Module>(
    bcx: &mut FunctionBuilder<'_>,
    module: &mut M,
    mat_sunk_id: cranelift_module::FuncId,
    escape: &EscapeAnalysis,
    virt_vars: &[Option<Vec<Variable>>],
    virt_kinds: &[Option<Vec<RegKind>>],
    regs_full: &[Variable],
    op_offsets: &[u32],
    cmp_op_idx: usize,
    kinds_snapshot: &mut [RegKind],
    head_proto: Gc<Proto>,
    // v1.3 Phase AOT Stage 7 sub-piece 2 — relocation context. `aot
    // == false` keeps the original JIT-time iconst behaviour;
    // `aot == true` routes interned-key pointers through the
    // `__luna_aot_strkey_slot_<hex>` data section. `defined_aot_data`
    // is per-lower-call dedup memory for `define_data` (only the
    // first occurrence of a given DataId may define its bytes).
    aot: bool,
    defined_aot_data: &mut std::collections::HashSet<DataId>,
) -> u32 {
    let mut count: u32 = 0;
    let empty: &[u32] = &[];
    let live: &[u32] = escape
        .live_at_op
        .get(cmp_op_idx)
        .map(|v| v.as_slice())
        .unwrap_or(empty);
    for &sid32 in live {
        let sid = sid32 as usize;
        if sid >= escape.sites.len() {
            continue;
        }
        let site = &escape.sites[sid];
        if site.state != EscapeState::Sinkable {
            continue;
        }
        let Some(vars) = virt_vars[sid].as_ref() else {
            continue;
        };
        let Some(kinds) = virt_kinds[sid].as_ref() else {
            continue;
        };
        let cap = site.array_cap as usize;
        if cap == 0 {
            continue;
        }
        // Site address in the trace's register window: caller-frame
        // address = site.a; inline-frame address = op_offsets[site.op_idx] + site.a.
        let off = op_offsets[site.op_idx] as usize;
        let reg_idx = off + site.a as usize;
        if reg_idx >= regs_full.len() {
            continue;
        }
        // Skip depth>0 sites when caller's snapshot is caller-window
        // only (max_stack sized) — the kind plumbing for those is
        // out of scope for the depth=0 cmp arm.
        if reg_idx >= kinds_snapshot.len() {
            continue;
        }
        let n_hash = site.hash_keys.len();
        // Array stack-alloc buffers; for cap=0 (hash-only site)
        // pass null pointers.
        let (raws_addr, kinds_addr) = if cap > 0 {
            let raws_ss = bcx.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
                cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
                (cap * 8) as u32,
                3,
            ));
            let kinds_ss = bcx.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
                cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
                cap as u32,
                0,
            ));
            for vi in 0..cap {
                let v = bcx.use_var(vars[vi]);
                bcx.ins().stack_store(v, raws_ss, (vi * 8) as i32);
                let tag = kind_to_raw_tag(kinds[vi]);
                let k = bcx.ins().iconst(types::I8, tag as i64);
                bcx.ins().stack_store(k, kinds_ss, vi as i32);
            }
            (
                bcx.ins().stack_addr(types::I64, raws_ss, 0),
                bcx.ins().stack_addr(types::I64, kinds_ss, 0),
            )
        } else {
            (
                bcx.ins().iconst(types::I64, 0),
                bcx.ins().iconst(types::I64, 0),
            )
        };
        // P12-S11-B-v2 — hash slot stack-alloc buffers (3 parallel
        // arrays: keys, raws, kinds). For each hash slot, fill from
        // virt_vars[cap + slot] + virt_kinds[cap + slot]; key ptr
        // comes from head_proto.consts[site.hash_keys[slot]] at
        // compile time (baked in as iconst).
        let (hash_keys_addr, hash_raws_addr, hash_kinds_addr) = if n_hash > 0 {
            let hash_keys_ss =
                bcx.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
                    cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
                    (n_hash * 8) as u32,
                    3,
                ));
            let hash_raws_ss =
                bcx.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
                    cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
                    (n_hash * 8) as u32,
                    3,
                ));
            let hash_kinds_ss =
                bcx.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
                    cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
                    n_hash as u32,
                    0,
                ));
            for vi in 0..n_hash {
                let slot = cap + vi;
                let v = bcx.use_var(vars[slot]);
                bcx.ins().stack_store(v, hash_raws_ss, (vi * 8) as i32);
                let tag = kind_to_raw_tag(kinds[slot]);
                let k = bcx.ins().iconst(types::I8, tag as i64);
                bcx.ins().stack_store(k, hash_kinds_ss, vi as i32);
                let const_idx = site.hash_keys[vi] as usize;
                let key_str = match head_proto.consts[const_idx] {
                    luna_core::runtime::Value::Str(s) => s,
                    _ => unreachable!(
                        "hash_keys must point at Str consts (validated by escape sweep)"
                    ),
                };
                let key_ptr_v = emit_str_key_arg(module, bcx, key_str, aot, defined_aot_data);
                bcx.ins()
                    .stack_store(key_ptr_v, hash_keys_ss, (vi * 8) as i32);
            }
            (
                bcx.ins().stack_addr(types::I64, hash_keys_ss, 0),
                bcx.ins().stack_addr(types::I64, hash_raws_ss, 0),
                bcx.ins().stack_addr(types::I64, hash_kinds_ss, 0),
            )
        } else {
            (
                bcx.ins().iconst(types::I64, 0),
                bcx.ins().iconst(types::I64, 0),
                bcx.ins().iconst(types::I64, 0),
            )
        };
        let cap_val = bcx.ins().iconst(types::I64, cap as i64);
        let n_hash_val = bcx.ins().iconst(types::I64, n_hash as i64);
        let mat_ref = module.declare_func_in_func(mat_sunk_id, bcx.func);
        let call = bcx.ins().call(
            mat_ref,
            &[
                cap_val,
                raws_addr,
                kinds_addr,
                n_hash_val,
                hash_keys_addr,
                hash_raws_addr,
                hash_kinds_addr,
            ],
        );
        let table_bits = bcx.inst_results(call)[0];
        bcx.def_var(regs_full[reg_idx], table_bits);
        kinds_snapshot[reg_idx] = RegKind::Table;
        count += 1;
    }
    count
}

/// P12-S8-C — does this op write the register at `R[A]` as its
/// sole / primary destination? Used by `const_fold_int_key` to
/// recognise the last writer of a key register. Conservative:
/// excludes ops that write multiple regs (Call, ForLoop) or that
/// don't write a reg at all (Set*, control flow). The const-fold
/// scan returns `None` on any unrecognised writer.
fn writes_target_a(op: Op) -> bool {
    matches!(
        op,
        Op::Move
            | Op::LoadI
            | Op::LoadF
            | Op::LoadK
            | Op::LoadNil
            | Op::Add
            | Op::Sub
            | Op::Mul
            | Op::Div
            | Op::IDiv
            | Op::Mod
            | Op::Pow
            | Op::BAnd
            | Op::BOr
            | Op::BXor
            | Op::Shl
            | Op::Shr
            | Op::Unm
            | Op::BNot
            | Op::Len
            | Op::NewTable
            | Op::GetI
            | Op::GetTable
            | Op::GetUpval
            | Op::GetField
            | Op::GetTabUp
            | Op::Closure
    )
}

/// P12-S8-C — walk backward from `set_table_idx` looking for the
/// most recent writer of `reg` at the same `inline_depth`. If it's
/// `LoadI sbx` with `sbx in 1..=cap`, the key is the const `sbx`.
/// `Move R[?] = R[src]` chains the search to `src` (up to
/// `MAX_STEPS` hops). Any other writer kills the trail → `None`.
fn const_fold_int_key(
    record: &TraceRecord,
    set_table_idx: usize,
    reg: u32,
    cap: u32,
) -> Option<u32> {
    const MAX_STEPS: usize = 8;
    let depth = record.ops[set_table_idx].inline_depth;
    let mut cur_reg = reg;
    let mut steps = 0;
    let mut j = set_table_idx;
    while j > 0 && steps < MAX_STEPS {
        j -= 1;
        steps += 1;
        let rop = &record.ops[j];
        if rop.inline_depth != depth {
            continue;
        }
        let inst = rop.inst;
        if inst.a() != cur_reg || !writes_target_a(inst.op()) {
            continue;
        }
        match inst.op() {
            Op::LoadI => {
                let sbx = inst.sbx();
                if sbx >= 1 && (sbx as u32) <= cap {
                    return Some(sbx as u32);
                }
                return None;
            }
            Op::Move => {
                cur_reg = inst.b();
                continue;
            }
            _ => return None,
        }
    }
    None
}

/// P12-S5-A — forward sweep over `record.ops[..effective_end]` to
/// classify NewTable sites. Conservative — over-escape is OK; the
/// rules below are correctness-preserving for any future emit (a
/// Sinkable site can always be heap-allocated; an Escaped site
/// MUST be).
///
/// Sweep rules:
/// - `Op::NewTable A=a B=cap C=0`, `cap > 0` → new `Sinkable` site
///   bound at `(depth, a)`. Hash-part (C != 0) or unknown cap (B == 0)
///   → unbind A, no site (S5-B+ handles a wider shape).
/// - `Op::SetList A=a B=cap C=0` writing through a bound `(depth, a)`
///   whose site's `array_cap == B` → array init, no escape on the
///   target. Source slots `A+1..=A+B` that themselves bind sites →
///   those sites escape (nested sinks not handled in S5-A).
/// - `Op::SetI` / `Op::SetTable`: value slot bound → escape (stored
///   into a (different) table).
/// - `Op::GetI` / `Op::GetTable` / `Op::Len`: read of B/A is fine;
///   write to A → unbind A.
/// - `Op::Move A=dst B=src`: src bound → conservatively escape src
///   (no aliasing). Always unbind A.
/// - `Op::Call A=fn B=narg+1`: any bound site in `[A+1..A+B-1]` (call
///   argument) or at A (the function being called) escapes. After
///   the Call, A holds the return value → unbind.
/// - `Op::Return1 A=a`: bound site at A escapes (carried to caller).
/// - `Op::Return0`: no value transfer.
/// - Cmp ops (`Op::Lt/Le/Eq/EqK`): every live binding escapes (the
///   cmp emits a side-exit and the interp may resume needing the
///   heap table).
/// - Other writer ops (arith / loads / GetUpval / GetField / etc.):
///   unbind A.
///
/// Terminator handling (the op at `effective_end`, if any):
/// - `TraceEnd::Call`: terminator's args + fn slot escape live bindings.
/// - `TraceEnd::ForLoop` / `TraceEnd::InlineAbort`: every live binding
///   escapes (loop exit / interp resume).
/// - `TraceEnd::Return`: `Return1` only → R[A] escapes; `Return0` is
///   a no-op.
fn escape_analyze(
    record: &TraceRecord,
    effective_end: usize,
    end_kind: Option<TraceEnd>,
    head_proto: Gc<Proto>,
) -> EscapeAnalysis {
    let max_stack = head_proto.max_stack as usize;
    let max_depth = (MAX_INLINE_DEPTH as usize) + 1;
    if max_stack == 0 {
        let upper0 = effective_end.min(record.ops.len());
        return EscapeAnalysis {
            sites: Vec::new(),
            op_actions: vec![None; upper0],
            live_at_op: vec![Vec::new(); upper0],
            accum_sites: Vec::new(),
            accum_live_at_op: Vec::new(),
        };
    }

    let mut bindings: Vec<Vec<Option<usize>>> = vec![vec![None; max_stack]; max_depth];
    let mut sites: Vec<AllocSite> = Vec::new();
    let upper = effective_end.min(record.ops.len());
    let mut op_actions: Vec<Option<OpAction>> = vec![None; upper];
    let mut live_at_op: Vec<Vec<u32>> = vec![Vec::new(); upper];

    fn mark_escape(sites: &mut [AllocSite], sid: usize) {
        if sites[sid].state == EscapeState::Sinkable {
            sites[sid].state = EscapeState::Escaped;
        }
    }
    fn lookup(bindings: &[Vec<Option<usize>>], depth: u8, reg: u32) -> Option<usize> {
        let d = depth as usize;
        let r = reg as usize;
        if d < bindings.len() && r < bindings[d].len() {
            bindings[d][r]
        } else {
            None
        }
    }
    fn unbind(bindings: &mut [Vec<Option<usize>>], depth: u8, reg: u32) {
        let d = depth as usize;
        let r = reg as usize;
        if d < bindings.len() && r < bindings[d].len() {
            bindings[d][r] = None;
        }
    }
    fn escape_all_live(bindings: &[Vec<Option<usize>>], sites: &mut [AllocSite]) {
        for row in bindings.iter() {
            for &slot in row.iter() {
                if let Some(sid) = slot {
                    mark_escape(sites, sid);
                }
            }
        }
    }

    for i in 0..upper {
        let cur_depth = record.ops[i].inline_depth as usize;
        // P12-S10-A — clear stale bindings from popped deeper
        // inline frames. After a Return*, control transitions
        // from depth N+1 back to depth N (the next op is at depth
        // N). The bindings rows for depths > N hold sites that
        // were tracked inside the now-popped frame; their
        // registers no longer point to live data. Without this
        // clear, live_at_op snapshots would include stale sites
        // and emit_materialize would index wrong inline windows.
        for d in (cur_depth + 1)..bindings.len() {
            for slot in bindings[d].iter_mut() {
                *slot = None;
            }
        }
        // P12-S5-C — snapshot live sunk bindings BEFORE the op
        // processes (each cmp emit uses live_at_op[cmp_idx] to
        // materialise the right virt slots).
        let mut live_snap: Vec<u32> = Vec::new();
        for row in bindings.iter() {
            for &slot in row.iter() {
                if let Some(sid) = slot {
                    live_snap.push(sid as u32);
                }
            }
        }
        live_at_op[i] = live_snap;

        let rop = &record.ops[i];
        let depth = rop.inline_depth;
        let ins = rop.inst;
        let a = ins.a();
        let op = ins.op();

        if (depth as usize) >= max_depth || (a as usize) >= max_stack {
            // The lowerer bails on OOB; skip silently here so the
            // sweep stays a side-effect-free analysis.
            continue;
        }

        match op {
            Op::NewTable => {
                // P12-S11-B-v1 — admit all NewTable shapes as
                // potential sites (was: only `b > 0, c = 0`). For
                // hash-only (b == 0) the array_cap is 0 and the
                // site only sunk-emits via SetField/GetField. For
                // mixed (b > 0, c > 0) array part sunk-emits as
                // before; hash slots accumulate via SetField scan.
                let cap = ins.b();
                let _c_hash = ins.c();
                unbind(&mut bindings, depth, a);
                let sid = sites.len();
                sites.push(AllocSite {
                    op_idx: i,
                    pc: rop.pc,
                    a,
                    inline_depth: depth,
                    array_cap: cap,
                    hash_keys: Vec::new(),
                    state: EscapeState::Sinkable,
                });
                bindings[depth as usize][a as usize] = Some(sid);
                // P12-S5-B — tag this op so emit can take the sunk
                // path (no NewTable helper call).
                op_actions[i] = Some(OpAction::NewTableSite {
                    site_idx: sid as u32,
                });
            }
            Op::SetList => {
                // P12-S9-C — B=0 form: use the recorder's var_count
                // snapshot (= top - A - 1 at the SetList op) as the
                // effective B. Otherwise the bytecode's B is the count.
                let b_bytecode = ins.b();
                let c = ins.c();
                let effective_b = if b_bytecode == 0 {
                    record.ops[i].var_count.unwrap_or_default()
                } else {
                    b_bytecode
                };
                if let Some(sid) = lookup(&bindings, depth, a) {
                    if c != 0 || ins.k() || sites[sid].array_cap != effective_b {
                        mark_escape(&mut sites, sid);
                    } else {
                        // P12-S5-B / P12-S9-C — supported form; tag for
                        // sunk emit (def_var virt slots from source
                        // registers). For B=0, the source range size
                        // is the recorded var_count.
                        op_actions[i] = Some(OpAction::SetListWrite {
                            site_idx: sid as u32,
                        });
                    }
                    for off in 1..=effective_b {
                        let src = a.wrapping_add(off);
                        if (src as usize) < max_stack
                            && let Some(src_sid) = lookup(&bindings, depth, src)
                        {
                            mark_escape(&mut sites, src_sid);
                        }
                    }
                }
            }
            Op::SetI => {
                // P12-S8-B — `R[A][B_imm] := R[C]`. Target slot
                // bound to a Sinkable site + key in 1..=cap →
                // tag SetISunkWrite (emit `def_var`s the value into
                // the matching virt slot). Otherwise the target
                // site escapes (helper-path SetI writes through the
                // real heap table). The value source slot escapes
                // unconditionally if it's bound — sinking a sunk
                // table into another sunk table's slot would need
                // pointer-aliasing the virt slot, deferred.
                let c = ins.c();
                if (c as usize) < max_stack
                    && let Some(src_sid) = lookup(&bindings, depth, c)
                {
                    mark_escape(&mut sites, src_sid);
                }
                if let Some(sid) = lookup(&bindings, depth, a) {
                    let key = ins.b();
                    if key >= 1 && key <= sites[sid].array_cap {
                        op_actions[i] = Some(OpAction::SetISunkWrite {
                            site_idx: sid as u32,
                            key,
                        });
                    } else {
                        // OOB key — sunk emit can't represent this
                        // write; helper path needed → site escapes.
                        mark_escape(&mut sites, sid);
                    }
                }
            }
            Op::SetField => {
                // P12-S11-B-v1 — `R[A][K[B]:string] := R[C]`. If R[A]
                // is bound to a Sinkable site AND R[C]'s value is not
                // itself a bound site (sinking a site into another
                // site's hash slot is out of scope), tag sunk: push
                // the key's const idx to site.hash_keys (if not
                // already there) and record the slot in OpAction.
                // Otherwise the target site escapes.
                let c = ins.c();
                let key_const_idx = ins.b();
                if (c as usize) < max_stack
                    && let Some(src_sid) = lookup(&bindings, depth, c)
                {
                    mark_escape(&mut sites, src_sid);
                }
                if let Some(sid) = lookup(&bindings, depth, a) {
                    // Find or insert the hash slot for this key.
                    let slot_opt = sites[sid]
                        .hash_keys
                        .iter()
                        .position(|&k| k == key_const_idx);
                    let slot = match slot_opt {
                        Some(s) => s,
                        None => {
                            let s = sites[sid].hash_keys.len();
                            sites[sid].hash_keys.push(key_const_idx);
                            s
                        }
                    };
                    op_actions[i] = Some(OpAction::SetFieldSunkWrite {
                        site_idx: sid as u32,
                        hash_slot: slot as u32,
                    });
                }
            }
            Op::GetField => {
                // P12-S11-B-v1 — `R[A] := R[B][K[C]:string]`. If R[B]
                // is bound to a Sinkable site AND the key has been
                // seen on this site (i.e. exists in site.hash_keys),
                // tag sunk read. Unknown key (first GetField for it
                // without a prior SetField) escapes the site —
                // reading uninitialised hash slot would be Nil at
                // runtime, but the trace's virt slot would hold
                // undefined value. Conservative escape.
                let b_reg = ins.b();
                let key_const_idx = ins.c();
                if (b_reg as usize) < max_stack
                    && let Some(sid) = lookup(&bindings, depth, b_reg)
                {
                    let slot_opt = sites[sid]
                        .hash_keys
                        .iter()
                        .position(|&k| k == key_const_idx);
                    match slot_opt {
                        Some(s) => {
                            op_actions[i] = Some(OpAction::GetFieldSunkRead {
                                site_idx: sid as u32,
                                hash_slot: s as u32,
                            });
                        }
                        None => {
                            mark_escape(&mut sites, sid);
                        }
                    }
                }
                unbind(&mut bindings, depth, a);
            }
            Op::SetTable => {
                // P12-S8-C — `R[A][R[B]] := R[C]`. Sunk emit requires
                // a compile-time-known int key. `const_fold_int_key`
                // walks back from this op looking for a LoadI (via a
                // Move chain) that pinned R[B]'s value to a literal
                // in 1..=cap. If found, tag SetTableSunkWrite (same
                // emit as SetI sunk path). Otherwise the target site
                // escapes (helper path runs through real heap table).
                // Value source still escapes if bound (same as SetI).
                let c = ins.c();
                if (c as usize) < max_stack
                    && let Some(src_sid) = lookup(&bindings, depth, c)
                {
                    mark_escape(&mut sites, src_sid);
                }
                if let Some(sid) = lookup(&bindings, depth, a) {
                    let cap = sites[sid].array_cap;
                    let key_reg = ins.b();
                    if let Some(key) =
                        const_fold_int_key(record, i, key_reg, cap)
                    {
                        op_actions[i] = Some(OpAction::SetTableSunkWrite {
                            site_idx: sid as u32,
                            key,
                        });
                    } else {
                        mark_escape(&mut sites, sid);
                    }
                }
            }
            Op::GetI => {
                // R[A] := R[B][C_imm]. Sunk-emit support: B must be
                // bound to a Sinkable site AND the immediate key C
                // must fall in `1..=array_cap`. Anything else: site
                // (if any) escapes.
                let b = ins.b();
                let c = ins.c();
                if (b as usize) < max_stack
                    && let Some(sid) = lookup(&bindings, depth, b)
                {
                    if c >= 1 && c <= sites[sid].array_cap {
                        op_actions[i] = Some(OpAction::GetIRead {
                            site_idx: sid as u32,
                            key: c,
                        });
                    } else {
                        // OOB or zero key — sunk emit can't represent
                        // this read; force heap path.
                        mark_escape(&mut sites, sid);
                    }
                }
                unbind(&mut bindings, depth, a);
            }
            Op::GetTable | Op::Len => {
                // GetTable: dynamic key — can't constant-fold. Len:
                // we know cap, but v1 doesn't sunk-emit Len. Either
                // way, if B (source table) is bound, escape.
                let b = ins.b();
                if (b as usize) < max_stack
                    && let Some(sid) = lookup(&bindings, depth, b)
                {
                    mark_escape(&mut sites, sid);
                }
                unbind(&mut bindings, depth, a);
            }
            Op::Move => {
                // P12-S8-A — Move is a binding alias: the dst reg
                // now references the same sunk site as src; src's
                // own binding stays. No escape — both regs are
                // interior-trace aliases, and downstream ops drive
                // escape via their own rules (SetI/SetTable/Len/Call
                // arg/Return1). Pre-S8-A this arm called
                // mark_escape(src_sid), which collapsed any sunk site
                // touched by Lua 5.5 frontend's `Move temp=R[t];
                // SetI temp[k]=v` lowering of `t[k]=v`.
                let b = ins.b();
                let src_sid = if (b as usize) < max_stack {
                    lookup(&bindings, depth, b)
                } else {
                    None
                };
                unbind(&mut bindings, depth, a);
                if let Some(sid) = src_sid {
                    bindings[depth as usize][a as usize] = Some(sid);
                }
            }
            Op::Call => {
                let b = ins.b();
                if b > 0 {
                    for off in 1..b {
                        let src = a.wrapping_add(off);
                        if (src as usize) < max_stack
                            && let Some(src_sid) = lookup(&bindings, depth, src)
                        {
                            mark_escape(&mut sites, src_sid);
                        }
                    }
                }
                if let Some(fn_sid) = lookup(&bindings, depth, a) {
                    mark_escape(&mut sites, fn_sid);
                }
                unbind(&mut bindings, depth, a);
            }
            Op::Return1 => {
                if let Some(sid) = lookup(&bindings, depth, a) {
                    mark_escape(&mut sites, sid);
                }
            }
            Op::Return0 => {}
            Op::Lt | Op::Le | Op::Eq | Op::EqK => {
                // P12-S5-C — cmp side-exits no longer auto-escape
                // live sunk sites. S5-C's `emit_materialize_*` runs
                // at every cmp side-exit emit point to materialize
                // the live virt slots into a heap `Gc<Table>` +
                // override the per-exit-tags entry to `Table`.
                //
                // depth=0 cmps materialize at the side-exit emit
                // path (Op::Lt/Le/Eq/EqK arm `else` branch — the
                // non-inline path). depth>0 cmps would need the
                // same machinery in the `per_exit_inline` arm; v1
                // demotes the site to Escaped instead (see the
                // `has_inline_cmp` gate in pre-emit demote).
            }
            Op::Add | Op::Sub | Op::Mul | Op::Div | Op::IDiv | Op::Mod
            | Op::Pow | Op::BAnd | Op::BOr | Op::BXor | Op::Shl | Op::Shr
            | Op::Unm | Op::BNot
            | Op::LoadI | Op::LoadF | Op::LoadK
            | Op::GetUpval | Op::GetTabUp | Op::Concat
            // P12-S7-A — Op::Closure writes a fresh LuaClosure
            // pointer into R[A]; no NewTable site lives there.
            // P12-S11-A — Op::GetField has its own arm above
            // (escapes R[B] receiver); not in this catch-all.
            | Op::Closure => {
                unbind(&mut bindings, depth, a);
            }
            Op::LoadNil => {
                // P12-S6-A2 — writes Nil to R[A..=A+B]. Each dest
                // slot loses its binding (a sunk site whose A is
                // overwritten with Nil is no longer reachable via
                // that slot — the actual table pointer is gone).
                let b = ins.b();
                for k in 0..=b {
                    let r = a.wrapping_add(k);
                    if (r as usize) < max_stack {
                        unbind(&mut bindings, depth, r);
                    }
                }
            }
            Op::Close => {
                // P12-S7-C — Op::Close A closes open upvals at slot
                // ≥ A. Open upvals point at vm.stack — the trace
                // can't keep them only in virt slots, so any live
                // sunk site whose slot ≥ A must escape (helper's
                // close_from reads vm.stack[s] to seal each upval).
                // Conservative: escape ALL live bindings (the helper
                // path also spills all live regs via emit, so this
                // matches the IR contract).
                escape_all_live(&bindings, &mut sites);
            }
            Op::Jmp | Op::ForLoop | Op::Return => {}
            _ => {
                unbind(&mut bindings, depth, a);
            }
        }
    }

    if let Some(end) = end_kind {
        if effective_end < record.ops.len() {
            let term = &record.ops[effective_end];
            let depth = term.inline_depth;
            let a = term.inst.a();
            let op = term.inst.op();
            let in_range = (depth as usize) < max_depth && (a as usize) < max_stack;
            match end {
                TraceEnd::Call => {
                    if in_range {
                        let b = term.inst.b();
                        if b > 0 {
                            for off in 1..b {
                                let src = a.wrapping_add(off);
                                if (src as usize) < max_stack
                                    && let Some(src_sid) = lookup(&bindings, depth, src)
                                {
                                    mark_escape(&mut sites, src_sid);
                                }
                            }
                        }
                        if let Some(fn_sid) = lookup(&bindings, depth, a) {
                            mark_escape(&mut sites, fn_sid);
                        }
                    }
                }
                TraceEnd::InlineAbort => {
                    escape_all_live(&bindings, &mut sites);
                }
                TraceEnd::ForLoop => {
                    // P12-S5-D — DO NOT auto-escape on a ForLoop
                    // terminator. ForLoop's IR side-exit fires on
                    // loop exit; interp resumes OUTSIDE the loop,
                    // where any `local t = {...}` declared inside
                    // the body is out of scope (parser frees the
                    // register slot at loop end). The dispatcher's
                    // exit-tag override (Sinkable slot → Untouched)
                    // keeps the slot reading as its entry tag.
                    //
                    // A mid-body cmp side-exit would still need
                    // materialise (interp resumes IN the loop body
                    // at the side-exit PC, where `t` may still be
                    // accessed) — the cmp arm in the body sweep
                    // already escapes live bindings, and the
                    // pre-emit `body_has_cmp` gate is a defensive
                    // backstop.
                }
                TraceEnd::Return => {
                    if matches!(op, Op::Return1)
                        && in_range
                        && let Some(sid) = lookup(&bindings, depth, a)
                    {
                        mark_escape(&mut sites, sid);
                    }
                }
                TraceEnd::SelfLink(_) => {
                    // P16-A — self-link close. Body loops with
                    // snapshot-restore (deepest-frame's window →
                    // head-frame's window). Every live binding at
                    // close must be marshalled back across the
                    // back-edge so the next iter sees a coherent
                    // window, same as InlineAbort's blanket escape.
                    escape_all_live(&bindings, &mut sites);
                }
                TraceEnd::DownRec { .. } => {
                    // v2.0 Track-R R3a — down-rec close. R3a routes
                    // the tail emit through R1's safe deopt path
                    // (dispatchable=false), so every live binding
                    // at close must be marshalled back into the
                    // caller window before the deopt return —
                    // same blanket-escape posture as SelfLink /
                    // InlineAbort. R3b's lowerer will keep this
                    // shape when it lifts to a real native back-edge:
                    // the retf-guard exit also returns through the
                    // caller window.
                    escape_all_live(&bindings, &mut sites);
                }
            }
        }
    }

    // P14-S14-B v0 — detect accumulator sites. The v0 stub returns
    // an empty Vec; v1+ will populate. Plumbed through here so the
    // EscapeAnalysis surface is stable for downstream consumers.
    let accum_sites = detect_accumulators(record, effective_end, head_proto);
    let accum_live_at_op = vec![Vec::new(); op_actions.len()];
    EscapeAnalysis {
        sites,
        op_actions,
        live_at_op,
        accum_sites,
        accum_live_at_op,
    }
}

/// P14-S14-B v1 — scan a trace for the `s = s .. v` 4-op idiom
/// emitted by Lua's frontend, then run an escape sweep over the
/// REAL accumulator slot.
///
/// The idiom:
/// ```text
///   pc i:   Move A=tmp   B=s_slot        ; load accumulator into temp
///   pc i+1: Move A=tmp+1 B=v_slot        ; load piece into temp+1
///   pc i+2: Concat A=tmp B=2             ; R[tmp] = R[tmp] .. R[tmp+1]
///   pc i+3: Move A=s_slot B=tmp          ; store accumulated back
/// ```
///
/// The `tmp` slot is local to the idiom; the REAL accumulator is
/// `s_slot` (the source of the first Move AND the destination of
/// the last Move). The escape sweep checks that `s_slot` has NO
/// uses outside this idiom in the trace body.
///
/// Algorithm:
/// 1. Walk `record.ops[..end]` for Op::Concat with `b() == 2` at
///    `inline_depth == 0`. For each, check the 3 surrounding ops
///    match the idiom.
/// 2. Filter call-triggered traces (need back-edge closure for the
///    accumulator-across-iterations semantic).
/// 3. For each idiom match, run an escape sweep over the trace body
///    — every reference to `s_slot` MUST be within the idiom (the
///    pre-Move src or post-Move dst) or the slot escapes.
/// 4. Return matched idioms as `AccumSite` entries.
///
/// v1 stops at analysis; v2+ wires the OpAction + buffered emit.
fn detect_accumulators(record: &TraceRecord, end: usize, _head_proto: Gc<Proto>) -> Vec<AccumSite> {
    use luna_core::vm::isa::Op;

    let upper = end.min(record.ops.len());
    // Need at least 4 ops to form the idiom.
    if upper < 4 {
        return Vec::new();
    }

    // Step 1: idiom scan. For each Concat at index `ci`, check the
    // 3 surrounding ops match.
    let mut candidates: Vec<(AccumSite, usize, usize, usize, usize)> = Vec::new();
    for ci in 2..upper.saturating_sub(1) {
        let concat_rop = &record.ops[ci];
        if !matches!(concat_rop.inst.op(), Op::Concat) {
            continue;
        }
        if concat_rop.inline_depth != 0 {
            continue;
        }
        if concat_rop.inst.b() != 2 {
            continue;
        }
        let tmp = concat_rop.inst.a();

        // Pre-Move 1: Move A=tmp B=s_slot
        let pre1 = &record.ops[ci - 2];
        if pre1.inline_depth != 0 || !matches!(pre1.inst.op(), Op::Move) || pre1.inst.a() != tmp {
            continue;
        }
        let s_slot = pre1.inst.b();

        // Pre-Move 2: Move A=tmp+1 B=v_slot
        let pre2 = &record.ops[ci - 1];
        if pre2.inline_depth != 0 || !matches!(pre2.inst.op(), Op::Move) || pre2.inst.a() != tmp + 1
        {
            continue;
        }
        let v_slot = pre2.inst.b();

        // Post-Move: Move A=s_slot B=tmp
        let post = &record.ops[ci + 1];
        if post.inline_depth != 0
            || !matches!(post.inst.op(), Op::Move)
            || post.inst.a() != s_slot
            || post.inst.b() != tmp
        {
            continue;
        }

        // P14-S14-B v4-fixup — both the accumulator slot and the
        // piece slot must be Str at recorder-fire time for the
        // buffered emit to be sound. `luna_jit_str_buf_extend`
        // unconditionally interprets the raw bits as a
        // `*const LuaStr`; a non-Str payload (e.g. Int(1) =
        // raw=1) would dereference address 0x1 → SIGSEGV. The
        // dispatcher's entry-tag guard (`src/vm/exec.rs:~5124`)
        // ensures runtime tags match `record.entry_tags`, so
        // gating on Str-at-recorder-fire is sufficient.
        // Pre-existing bug: bisect confirmed `0055c22` (S14-B
        // v4-part2 real emit) — the snapshot would catch any
        // tag, and the buffered path would extend that raw into
        // the helper. Workload that exposed it:
        // `trace_jit_s12_step_c_v3::ipairs_mixed_tag_array_deopts
        // _no_garbage` with `{'a', 1, 'c'}`.
        let entry_tags = &record.entry_tags;
        let s_tag = entry_tags.get(s_slot as usize).copied();
        let v_tag = entry_tags.get(v_slot as usize).copied();
        if s_tag != Some(luna_core::runtime::value::raw::STR)
            || v_tag != Some(luna_core::runtime::value::raw::STR)
        {
            continue;
        }
        candidates.push((
            AccumSite {
                op_idx: ci,
                pc: concat_rop.pc,
                accum_slot: s_slot,
                piece_slot: v_slot,
                inline_depth: 0,
                state: BufferState::Bufferable,
            },
            ci - 2, // pre1 idx
            ci - 1, // pre2 idx
            ci,     // concat idx
            ci + 1, // post idx
        ));
    }
    if candidates.is_empty() {
        return Vec::new();
    }

    // Step 2: require the trace to be a back-edge loop (closed AND
    // not call-triggered) so the accumulator semantic ("survives
    // across iterations") holds. Call-triggered traces close on
    // re-entry of head_pc, which doesn't guarantee a loop semantic.
    if !record.closed || record.is_call_triggered {
        return Vec::new();
    }

    // Step 3: per-candidate escape sweep over body ops outside the
    // idiom. The 4 idiom op indices (pre1/pre2/concat/post) are
    // skipped — every other op must NOT reference the accumulator
    // slot, or it's NonBuffered.
    let mut out: Vec<AccumSite> = Vec::with_capacity(candidates.len());
    for (mut site, pre1, pre2, concat, post) in candidates {
        for i in 0..upper {
            if i == pre1 || i == pre2 || i == concat || i == post {
                continue;
            }
            let rop = &record.ops[i];
            if rop.inline_depth != 0 {
                continue;
            }
            let ins = rop.inst;
            let op = ins.op();
            let a = ins.a();
            let b = ins.b();
            let c = ins.c();
            let reads_slot = match op {
                Op::Move => b == site.accum_slot,
                Op::Add
                | Op::Sub
                | Op::Mul
                | Op::Div
                | Op::Mod
                | Op::Pow
                | Op::IDiv
                | Op::BAnd
                | Op::BOr
                | Op::BXor
                | Op::Shl
                | Op::Shr => b == site.accum_slot || c == site.accum_slot,
                Op::Unm | Op::BNot | Op::Not | Op::Len => b == site.accum_slot,
                Op::Eq | Op::Lt | Op::Le => a == site.accum_slot || b == site.accum_slot,
                Op::EqK | Op::Test | Op::TestSet => a == site.accum_slot,
                Op::Concat => {
                    let n = ins.b();
                    let end_op = a.saturating_add(n);
                    (a..end_op).any(|r| r == site.accum_slot)
                }
                Op::Call | Op::TailCall => {
                    let lo = a;
                    let hi = lo.saturating_add(b.max(1));
                    site.accum_slot >= lo && site.accum_slot < hi
                }
                Op::Return | Op::Return1 => a == site.accum_slot,
                Op::Return0 => false,
                Op::SetI | Op::SetTable | Op::SetField | Op::SetUpval | Op::SetTabUp => {
                    a == site.accum_slot || c == site.accum_slot
                }
                Op::GetI | Op::GetTable | Op::GetField | Op::GetTabUp | Op::GetUpval => {
                    b == site.accum_slot
                }
                Op::SetList => {
                    let lo = a;
                    let hi = lo.saturating_add(b.max(1));
                    site.accum_slot >= lo && site.accum_slot < hi
                }
                _ => false,
            };
            let writes_slot = match op {
                Op::LoadNil => {
                    let lo = a;
                    let hi = lo.saturating_add(b);
                    site.accum_slot >= lo && site.accum_slot <= hi
                }
                Op::Move
                | Op::LoadI
                | Op::LoadF
                | Op::LoadK
                | Op::LoadKx
                | Op::LoadFalse
                | Op::LFalseSkip
                | Op::LoadTrue
                | Op::GetUpval
                | Op::GetTabUp
                | Op::GetTable
                | Op::GetI
                | Op::GetField
                | Op::NewTable
                | Op::Add
                | Op::Sub
                | Op::Mul
                | Op::Div
                | Op::Mod
                | Op::Pow
                | Op::IDiv
                | Op::BAnd
                | Op::BOr
                | Op::BXor
                | Op::Shl
                | Op::Shr
                | Op::Unm
                | Op::BNot
                | Op::Not
                | Op::Len
                | Op::Concat
                | Op::Call
                | Op::TailCall
                | Op::TestSet
                | Op::ForLoop
                | Op::ForPrep
                | Op::TForCall => a == site.accum_slot,
                _ => false,
            };
            if reads_slot || writes_slot {
                site.state = BufferState::NonBuffered;
                break;
            }
        }
        out.push(site);
    }
    out
}

/// P15-A v2-E — per-op (reads, writes) slot analysis. Returns the
/// slot indices an op READS from and WRITES to in the caller's
/// register window. Conservative for unknown / not-yet-classified
/// ops: read range is widened (assume reads everything in the
/// range we're aware of), writes is empty — so the safety check
/// (`child.live_in ∩ parent.body_writes`) errs on the side of
/// bailing the side trace compile.
///
/// Caller is responsible for applying `inline_depth` offsets if
/// the op lives in a depth>0 inlined frame.
pub fn op_reads_writes(inst: luna_core::vm::isa::Inst) -> (Vec<u32>, Vec<u32>) {
    use luna_core::vm::isa::Op;
    let a = inst.a();
    let b = inst.b();
    let c = inst.c();
    let k = inst.k();
    match inst.op() {
        Op::Move => (vec![b], vec![a]),
        Op::LoadI | Op::LoadF | Op::LoadK | Op::LoadKx => (vec![], vec![a]),
        Op::LoadFalse | Op::LoadTrue | Op::LFalseSkip => (vec![], vec![a]),
        Op::LoadNil => {
            // R[A..=A+B] := nil
            let mut w = Vec::with_capacity((b + 1) as usize);
            for i in 0..=b {
                w.push(a + i);
            }
            (vec![], w)
        }
        Op::GetUpval => (vec![], vec![a]),
        Op::SetUpval => (vec![a], vec![]),
        Op::GetTabUp => (vec![], vec![a]),
        Op::GetTable => (vec![b, c], vec![a]),
        Op::GetI => (vec![b], vec![a]),
        Op::GetField => (vec![b], vec![a]),
        Op::SetTabUp => {
            // upval[b][const_b_or_R[B]] = R[C] / K[C]
            let mut r = Vec::new();
            if !k {
                r.push(c);
            }
            (r, vec![])
        }
        Op::SetTable => {
            let mut r = vec![a, b];
            if !k {
                r.push(c);
            }
            (r, vec![])
        }
        Op::SetI => {
            let mut r = vec![a];
            if !k {
                r.push(c);
            }
            (r, vec![])
        }
        Op::SetField => {
            let mut r = vec![a];
            if !k {
                r.push(c);
            }
            (r, vec![])
        }
        Op::NewTable => (vec![], vec![a]),
        Op::SelfOp => (vec![b], vec![a, a + 1]),
        Op::Add
        | Op::Sub
        | Op::Mul
        | Op::Mod
        | Op::Pow
        | Op::Div
        | Op::IDiv
        | Op::BAnd
        | Op::BOr
        | Op::BXor
        | Op::Shl
        | Op::Shr => (vec![b, c], vec![a]),
        Op::Unm | Op::BNot | Op::Not | Op::Len => (vec![b], vec![a]),
        Op::Concat => {
            // R[A] := concat(R[A..A+B-1])
            let mut r = Vec::with_capacity(b as usize);
            for i in 0..b {
                r.push(a + i);
            }
            (r, vec![a])
        }
        Op::Close | Op::Tbc => (vec![], vec![]),
        Op::Jmp | Op::ExtraArg => (vec![], vec![]),
        Op::Eq | Op::Lt | Op::Le => (vec![a, b], vec![]),
        Op::EqK => (vec![a], vec![]),
        Op::Test => (vec![a], vec![]),
        Op::TestSet => (vec![b], vec![a]),
        Op::Call => {
            // R[A..A+B-1] are args (incl. fn at R[A]); writes R[A..A+C-1]
            // B=0 means variable (top); C=0 means variable. Conservative:
            // assume B,C up to a reasonable cap (use observed values).
            let nargs = if b == 0 { 0 } else { b - 1 };
            let nres = if c == 0 { 0 } else { c - 1 };
            let mut r = vec![a];
            for i in 1..=nargs {
                r.push(a + i);
            }
            let mut w = Vec::with_capacity(nres as usize);
            for i in 0..nres {
                w.push(a + i);
            }
            (r, w)
        }
        Op::TailCall => {
            let nargs = if b == 0 { 0 } else { b - 1 };
            let mut r = vec![a];
            for i in 1..=nargs {
                r.push(a + i);
            }
            (r, vec![])
        }
        Op::Return => {
            // R[A..A+B-2] returned
            let n = if b == 0 { 1 } else { b - 1 };
            let mut r = Vec::with_capacity(n as usize);
            for i in 0..n {
                r.push(a + i);
            }
            (r, vec![])
        }
        Op::Return0 => (vec![], vec![]),
        Op::Return1 => (vec![a], vec![]),
        Op::ForLoop => {
            // R[A+1] = count, R[A] = idx, R[A+2] = step, R[A+3] = ctrl
            // Reads R[A], R[A+1], R[A+2]; writes R[A], R[A+1], R[A+3].
            (vec![a, a + 1, a + 2], vec![a, a + 1, a + 3])
        }
        Op::ForPrep => {
            // Sets up the for loop: reads init/limit/step, writes idx/count/ctrl.
            (vec![a, a + 1, a + 2], vec![a, a + 1, a + 3])
        }
        Op::TForPrep => (vec![], vec![]),
        Op::TForCall => {
            // R[A+4], R[A+5], ..., R[A+3+C] := R[A](R[A+1], R[A+2])
            let mut w = Vec::with_capacity(c as usize);
            for i in 0..c {
                w.push(a + 4 + i);
            }
            (vec![a, a + 1, a + 2], w)
        }
        Op::TForLoop => {
            // If R[A+4] ~= nil: R[A+2] = R[A+4]; pc -= Bx
            (vec![a + 4], vec![a + 2])
        }
        Op::SetList => {
            // R[A] is the table; R[A+1..A+B] are values to set.
            let n = if b == 0 { 0 } else { b };
            let mut r = vec![a];
            for i in 1..=n {
                r.push(a + i);
            }
            (r, vec![])
        }
        Op::Closure => (vec![], vec![a]),
        Op::Vararg | Op::GetVarg => {
            // Writes a variable count starting at R[A]. Conservative: just write R[A].
            (vec![], vec![a])
        }
        Op::VargIdx => (vec![c], vec![a]),
        Op::ErrNNil => (vec![a], vec![]),
    }
}

/// P15-A v2-E — compute the slot indices an op WRITES in the
/// caller's window, with the op's inline depth offset applied.
/// Used by `compute_body_writes` and `compute_live_in_slots`.
fn op_writes_at_offset(rop: &RecordedOp, op_offset: u32) -> Vec<u32> {
    let (_r, w) = op_reads_writes(rop.inst);
    w.into_iter().map(|s| op_offset + s).collect()
}

fn op_reads_at_offset(rop: &RecordedOp, op_offset: u32) -> Vec<u32> {
    let (r, _w) = op_reads_writes(rop.inst);
    r.into_iter().map(|s| op_offset + s).collect()
}

/// P15-A v2-E — compute the parent body's slot-write set. Walks
/// `record.ops`, applying each op's `inline_depth` offset, and
/// returns a sorted unique list of slot indices that ANY op writes.
/// Stored on `CompiledTrace.body_writes` so child side traces can
/// intersect against it at compile time.
pub fn compute_body_writes(record: &TraceRecord, op_offsets: &[u32]) -> Vec<u32> {
    let mut s: std::collections::BTreeSet<u32> = std::collections::BTreeSet::new();
    for (i, rop) in record.ops.iter().enumerate() {
        let off = op_offsets.get(i).copied().unwrap_or(0);
        for w in op_writes_at_offset(rop, off) {
            s.insert(w);
        }
    }
    s.into_iter().collect()
}

/// P15-A v2-E — compute the side trace's "live-in" slot set: slots
/// READ by some op without any prior write to the same slot within
/// `record.ops`. These are the values the side trace consumes from
/// its entry state (= what the parent wrote to reg_state at its
/// exit). If any live-in slot is ALSO in the parent's body_writes,
/// the side trace is UNSAFE to compile with internal looping (each
/// iter would re-read parent's stale write — see the s12_step_b
/// `Move R[1] = R[12]` bug).
pub fn compute_live_in_slots(record: &TraceRecord, op_offsets: &[u32]) -> Vec<u32> {
    let mut written: std::collections::HashSet<u32> = std::collections::HashSet::new();
    let mut live_in: std::collections::BTreeSet<u32> = std::collections::BTreeSet::new();
    for (i, rop) in record.ops.iter().enumerate() {
        let off = op_offsets.get(i).copied().unwrap_or(0);
        // Reads first — if a slot hasn't been written by a prior op,
        // it's live-in.
        for r in op_reads_at_offset(rop, off) {
            if !written.contains(&r) {
                live_in.insert(r);
            }
        }
        // Then mark writes (the op's writes happen "after" its reads
        // for purposes of subsequent ops).
        for w in op_writes_at_offset(rop, off) {
            written.insert(w);
        }
    }
    live_in.into_iter().collect()
}

/// Owner of one compiled trace's mmap'd code. Drop releases the
/// pages, so the handle is parked on the owning `Vm`'s
/// `storage.trace_handles` Vec, keeping the entry fn pointer
/// callable for the lifetime of that `Vm`.
///
/// v2.0 Track J sub-step J-B Phase F — was thread-local
/// `TRACE_JIT_HANDLES`. Mirrors the method JIT's `JitHandle` /
/// `storage.cache_handles` pattern (`jit_backend/mod.rs`).
pub struct TraceHandle {
    // v2.0 Track J sub-step J-D — sleeve `JITModule` in J-A's
    // `SendJitModule` newtype so the module's `Send` claim is
    // expressed at the field type, not by an `unsafe impl Send` on
    // the outer struct. The wrapper is `repr(Rust)` newtype with
    // `Deref<Target = JITModule>` so any internal call site that
    // touched `handle._module.<method>` still resolves through Deref.
    _module: super::SendJitModule,
    _entry_raw: *const u8,
}

// SAFETY: `SendJitModule` (J-A) is `Send` because luna only ever
// constructs `JITModule` with the default `SystemMemoryProvider`
// (which is `Send`). `_entry_raw: *const u8` is `!Send` by default;
// the manual `unsafe impl Send for TraceHandle` therefore stays
// load-bearing for the outer struct, but the J-A wrapper localizes
// the JITModule-side soundness reasoning to one place.
//
// v2.0 Track J sub-step J-E (audit): `_entry_raw` addresses mcode
// in `_module`'s mmap'd page. Because `_module` ships with the
// handle (the handle owns it by-value as a `SendJitModule`), the
// pointer remains dereferenceable on whichever OS thread the
// handle lands on after a Vm move. The pointer is read-only on
// the dispatch hot path (transmuted to an `extern "C"` fn and
// called); no aliasing concerns. Per-dispatch `JIT_VM` / `JIT_CL`
// TLS slots are scoped via J-D's `scoped_jit_vm_rebind` RAII so
// any thread that calls into the dispatcher re-arms its own slot.
// Mirror impl: `unsafe impl Send for JitHandle` at
// `jit_backend/mod.rs` just after the `JitHandle` struct.
//
// SAFETY: called only from Cranelift-emitted JIT code under an active JitVmGuard; the guard guarantees JIT_VM TLS holds a live &mut Vm for the dispatch window.
unsafe impl Send for TraceHandle {}

impl TraceHandle {
    /// v2.0 Track J sub-step J-D — `#[doc(hidden)]` accessor returning
    /// the parked `_module` borrowed at the `SendJitModule` newtype.
    /// Mirror of `JitHandle::__j_d_module`; lets
    /// `tests/j_d_scoped_rebind_and_sleeve.rs` statically assert the
    /// field type.
    #[doc(hidden)]
    #[inline]
    pub fn __j_d_module(&self) -> &super::SendJitModule {
        &self._module
    }
}

// v2.0 Track J sub-step J-B Phase F — `TRACE_JIT_HANDLES` was a
// `thread_local!<Vec<TraceHandle>>` here. Migrated to
// `Vm.jit.storage.trace_handles`. Compiled fn pointers stay callable
// for the lifetime of the owning `Vm` instead of the thread.

/// Step 5 op whitelist. Anything outside this set bails the lowerer
/// to `None`, leaving the recorder to drop the trace.
///
/// - `Move` — `R[A] = R[B]`. Type-agnostic (just copies 8-byte
///   payload), so it composes with later steps when Float arrives.
/// - `Add / Sub / Mul` — Int-Int arithmetic. The lowerer assumes
///   the recorded operand types were Int; without value guards
///   (S2.C / S3 dispatcher territory), the caller must ensure live
///   reg values match the recorded types before invoking the trace.
/// - `Jmp` — emits no IR. Two valid roles:
///   (1) consumed-by-cmp — paired with a preceding `Lt / Le / Eq`
///   at `cmp.pc + 1`; the cmp's brif's "continue" branch already
///   represents control passing past the Jmp, so emitting jump
///   IR would be wrong.
///   (2) trailing back-edge — the last op of the trace, closing
///   the loop back to `head_pc`. The tail's `return iconst(head_pc)`
///   carries the control transfer; no IR for the Jmp itself.
///   A Jmp in any other position bails the trace.
/// - `Lt / Le / Eq` — Int-Int comparison + side-exit guard. Each
///   cmp must be followed by a `Jmp` at `cmp.pc + 1` encoding the
///   "took the Jmp" direction. The lowerer emits `icmp` + `brif`:
///   a runtime mismatch stores reg state back and returns the
///   failing PC.
/// - `NewTable` — `R[A] = {}`. Lowered as a cranelift call to
///   `luna_jit_new_table`. Step 4 ignores the asize / hsize hints
///   (PUC's `Op::NewTable A B C` encodes them in B/C); step-4
///   follow-up can swap in `luna_jit_new_table_sized` when a
///   `for` loop pre-fold is detected.
/// - `SetI / GetI` — `R[A][B_imm] = R[C_reg]` / `R[A] = R[B_reg][C_imm]`,
///   where the key is the bytecode immediate. Lowered as
///   `luna_jit_table_set_int` / `luna_jit_table_get_int`. Both
///   helpers park `vm.jit.pending_err` on a metatable hit so the
///   dispatcher can deopt — semantics that bypass `__index` /
///   `__newindex` would silently miscompile.
/// - `Len` — `R[A] = #R[B]`. Lowered as `luna_jit_table_len`,
///   which also short-circuits on a metatable (5.4+ `__len`).
/// - `Call` — *trace-truncating* side-exit. The first `Op::Call`
///   in the recorded ops ends the trace: every op before it gets
///   normal IR, the Call emits a side-exit at its own PC (interp
///   resumes with full reg state), and every recorded op after it
///   is dropped. The post-Call ops in `record.ops` are the callee
///   body / Return / post-call continuation that the recorder
///   naturally inlines (S1.C always records `inline_depth = 0`),
///   but the step-5 lowerer can't tell them apart from the outer
///   frame and refuses to emit them; S4 (real inline) will handle
///   that. The Call is **not** verified to be self-recursive at
///   this step — the lowerer trusts the recorder to only feed
///   sound recursive patterns.
/// Which terminating op (if any) sits at the trace's effective
/// tail position. See the comment block in
/// [`try_compile_trace_with_options`] for the contracts on each.
#[derive(Clone, Copy, Debug)]
enum TraceEnd {
    Call,
    ForLoop,
    /// P12-S4-step3b — the trace's inline-recursion path hit something
    /// the lowerer can't continue past (ForLoop@d>0, a non-self
    /// Call@d>0, depth past MAX_INLINE_DEPTH, or a proto mismatch).
    /// emit `ops[..i]` normally, then close the tail with a
    /// store-back + return of `record.ops[i].pc`. Dispatchable is
    /// forced false because the interp can't resume at that PC
    /// without first materialising the depth>0 CallFrames — that's
    /// step 4's job. cmp@d>0 used to land here too but step4b-C-2
    /// now emits a real side-exit via the frame-mat helper.
    InlineAbort,
    /// P12-S4-step4b-C-2 — `Op::Return0` / `Op::Return1` at depth=0
    /// terminates the trace (the caller frame unwinds). Treat as a
    /// truncation point: emit `ops[..i]` normally, then store back
    /// the caller window + return `record.ops[i].pc`. The interp
    /// re-executes the Return instruction with the correct PC. Same
    /// shape as `TraceEnd::Call` but emitted by a different op, so
    /// kept as a separate variant for the tail dispatch.
    Return,
    /// P16-A — recorder detected self-recursion via the cycle catch
    /// (same-proto ancestor count > [`RECUNROLL_THRESHOLD`] at the
    /// head_pc on head_proto). The trace body covers the inlined
    /// recursion levels; the lowerer's tail emits a snapshot-restore
    /// (copy the deepest-inlined-frame's window into the head frame's
    /// window) + branch to `body_loop`. Each iter absorbs
    /// `RECUNROLL_THRESHOLD + 1` recursion levels. Dispatchable is
    /// `true` (DOES NOT pin `is_inline_abort_close`); the depth>0
    /// ops in the body are intentional inline content.
    SelfLink(SelfRecKind),
    /// v2.0 Track-R R3a — recorder detected a down-recursion close
    /// shape: a depth>0 `Op::Return` fired during recording AND the
    /// `rec.retfs` chain showed the trace bouncing in-and-out of the
    /// same caller-proto past [`RECUNROLL_THRESHOLD`]. Mirrors
    /// LuaJIT's `LJ_TRLINK_DOWNREC` close (`lj_record.c:912
    /// lj_trace_err(LJ_TRERR_DOWNREC)` → `lj_trace.c:570
    /// trace_downrec` → restart-at-Return-PC). Routed BEFORE the
    /// `SelfLink` arm in the `end_idx` picker so depth>0 Return
    /// closes win over depth>0 self-link cycles.
    ///
    /// `return_pc` is the PC the inlined frame is unwinding to —
    /// the future stitch-entry head_pc that R3b's lowerer will bake
    /// into the retf-guard sequence (`asm_retf` equivalent).
    /// `target_proto_id` carries the target proto's `Gc::as_ptr()`
    /// as a raw `usize` so this enum stays `Copy` for the existing
    /// `end_idx_opt: Option<(usize, TraceEnd)>` plumbing. The
    /// matching `Gc<Proto>` lives on `TraceRecord.downrec_close.
    /// target_proto` (not erased); the lowerer cross-references the
    /// two when emitting the guard.
    ///
    /// **R3a constraint**: the lowerer still falls through to R1's
    /// safe `dispatchable=false` deopt-tail (`"self-link-retf-r1"`
    /// label retained) when this arm fires. R3b lifts that to a real
    /// native back-edge by emitting the retf-guard + stitch sentinel.
    DownRec {
        /// PC the Return is unwinding to (caller's resume PC).
        return_pc: u32,
        /// Caller proto's `Gc::as_ptr()` as `usize` (kept opaque so
        /// `TraceEnd` stays `Copy`). The lowerer reads
        /// `TraceRecord.downrec_close.target_proto` for the real
        /// `Gc<Proto>`.
        target_proto_id: usize,
        /// `from_depth - to_depth` at the moment the catch tripped.
        /// Always `1` today.
        depth_delta: u8,
    },
}

/// Direction the cmp/Jmp pair took in the recording. Both
/// directions can compile, but the brif's predicate and the
/// side-exit PC flip between them.
#[derive(Clone, Copy, Debug)]
enum CmpDir {
    /// Recorded: cmp matched K → no pc++ → Jmp executed. Next
    /// recorded op is the Jmp at `cmp_pc + 1`; consumed_by_cmp
    /// marks it. Side-exit PC = `cmp_pc + 2` (interp pc++).
    /// Standard repeat-until / `for` exit-cmp shape.
    TookJmp,
    /// Recorded: cmp didn't match K → pc++ → Jmp skipped. Next
    /// recorded op is the body op at `cmp_pc + 2`. Side-exit PC
    /// = the Jmp's target. Standard `while cond do` body-entry
    /// shape.
    SkippedJmp,
}

fn is_whitelisted_step5(op: Op) -> bool {
    matches!(
        op,
        Op::Move
            | Op::Add
            | Op::Sub
            | Op::Mul
            | Op::Div
            | Op::Pow
            | Op::IDiv
            | Op::Mod
            | Op::BAnd
            | Op::BOr
            | Op::BXor
            | Op::Shl
            | Op::Shr
            | Op::Unm
            | Op::BNot
            | Op::Jmp
            | Op::Lt
            | Op::Le
            | Op::Eq
            | Op::EqK
            | Op::NewTable
            | Op::GetI
            | Op::GetTable
            | Op::SetI
            | Op::SetTable
            | Op::SetList
            | Op::Len
            | Op::Call
            | Op::ForLoop
            | Op::LoadI
            | Op::LoadF
            | Op::LoadK
            // P12-S6-A2 — Op::LoadNil writes Nil to R[A..=A+B].
            // Emit: iconst(0) + def_var per slot + current_kinds[slot]
            // = RegKind::Nil. ExitTag::Nil (S6-A1) carries the Nil
            // through restore so non-Nil entry slots get repacked
            // as Value::Nil rather than mis-typed.
            | Op::LoadNil
            // P12-S7-A — Op::Closure creates `R[A] := closure(proto[Bx])`.
            // Emit: call `luna_jit_op_closure(bx)` (shared-upval path
            // only; in_stack upvals bail compile in pre-emit). Result
            // is the Gc<LuaClosure> raw payload; current_kinds =
            // RegKind::Closure → ExitTag::Closure on side-exit restore.
            | Op::Closure
            // P12-S7-C — Op::Close closes open upvals at slot ≥ A.
            // Emit: pre-Close spill of all live regs ≥ A, then
            // call `luna_jit_op_close(a)` returning 0 (continue) or
            // 1 (deopt). Deopt block writes store_back + returns
            // close_pc so interp redoes the Op::Close. Helper's
            // close_from is idempotent on the deopt path (open
            // upvals already popped).
            | Op::Close
            // P12-S4-step2b — Op::GetUpval reads the trace head
            // closure's upvals[idx] via the `luna_jit_upval_get`
            // helper (the dispatcher's enter_jit pins JIT_CL).
            | Op::GetUpval
            // GetTabUp / GetField are admitted ONLY inside a math
            // fold; the pre-emit pass enforces that gate via
            // `folded_math[i]`.
            | Op::GetTabUp
            | Op::GetField
            // P12-S11-A — Op::SetField writes `R[A][K[B]:string] = R[C]`.
            // Helper-path emit calls luna_jit_table_set_field with the
            // string key's Gc<LuaStr> raw ptr baked into IR.
            | Op::SetField
            // P12-S12-A — Op::Test gates `if x then ...` branches
            // when x isn't a comparison. Followed by Op::Jmp (taken
            // or skipped depending on R[A] truthiness vs K). v1
            // only handles kind-known truthy/falsy via compile-time
            // const fold (no IR — recorded direction is provably
            // stable); RegKind::Unset bails compile.
            | Op::Test
            // P12-S12-A-v2 — Op::TestSet is `if R[B].truthy()==K
            // then R[A]=R[B] else pc++`. Same kind-fold approach
            // as Op::Test (truthy of R[B]); on test-pass branch
            // (TookJmp recorded), emit a Move-style def_var
            // R[A] = R[B].
            | Op::TestSet
            // P12-S12-B-v2 — generic-for ops. TForPrep is a forward
            // pc-bump emitted before the body (head_pc = body_top,
            // so recorder never actually sees TForPrep in record —
            // whitelist only as a defensive arm). TForCall calls
            // the iterator via `luna_jit_op_tforcall` helper.
            // TForLoop terminates the trace at its back-edge,
            // handled in the tail emit (same TraceEnd::ForLoop
            // arm as Op::ForLoop, dispatch branches on inst.op()).
            | Op::TForPrep
            | Op::TForCall
            | Op::TForLoop
            // P12-S12-C v1 — Op::Concat A B does an N-operand
            // right-associative fold over `R[A..A+B-1]`, writing
            // the resulting string to R[A]. Trace emit spills the
            // operand window to vm.stack and calls
            // `luna_jit_op_concat(A, B)` helper which runs
            // concat_run + detects/deopts on the __concat
            // metamethod path. Helper-path equivalent to interp
            // (perf wash); architectural completeness only — real
            // perf wins live in P14 string subsystem.
            | Op::Concat
    )
}

/// Find a register's kind by walking back from `idx-1` through the
/// already-emitted writers in `current_kinds`. Avoids re-deriving
/// the kind from scratch on every operand access.
fn k_op(current_kinds: &[RegKind], reg: u32) -> RegKind {
    *current_kinds.get(reg as usize).unwrap_or(&RegKind::Unset)
}

/// Cast a Variable's i64 payload into f64 if its kind is Float.
fn use_var_f64(bcx: &mut FunctionBuilder<'_>, regs: &[Variable], reg: u32) -> Value {
    let raw = bcx.use_var(regs[reg as usize]);
    bcx.ins().bitcast(types::F64, MemFlags::new(), raw)
}

/// Store an f64 SSA value into a Variable as i64 bits.
fn def_var_f64(bcx: &mut FunctionBuilder<'_>, var: Variable, val_f64: Value) {
    let bits = bcx.ins().bitcast(types::I64, MemFlags::new(), val_f64);
    bcx.def_var(var, bits);
}

/// Emit a store-back of every `regs[i]` Variable to
/// `reg_state[i * 8]`, followed by `return iconst(pc)`. Used by both
/// the clean-close tail (pc = head_pc) and every cmp's side-exit
/// block (pc = failing_pc).
/// P12-S7-C — central dispatch for `t[key] = v` helper-path emit.
/// Picks the right specialized helper by the source register's
/// `RegKind`. Without the kind-aware dispatch a Closure / Table /
/// Float src would be silently wrapped as `Value::Int(raw_bits)`
/// by the legacy `set_int` helper.
fn emit_table_set<M: Module>(
    bcx: &mut FunctionBuilder<'_>,
    module: &mut M,
    set_int_id: cranelift_module::FuncId,
    set_nil_id: cranelift_module::FuncId,
    set_raw_id: cranelift_module::FuncId,
    t: Value,
    key: Value,
    val_kind: RegKind,
    val_var: Variable,
) {
    use luna_core::runtime::value::raw;
    match val_kind {
        RegKind::Nil => {
            let f = module.declare_func_in_func(set_nil_id, bcx.func);
            bcx.ins().call(f, &[t, key]);
        }
        // Int / Unset → set_int (legacy fast path; synth tests +
        // un-snapshotted slots default to Int payload, which matches
        // the pre-S7-C behaviour). Production traces with proper
        // kind tracking pin Int explicitly here.
        RegKind::Int | RegKind::Unset => {
            let v = bcx.use_var(val_var);
            let f = module.declare_func_in_func(set_int_id, bcx.func);
            bcx.ins().call(f, &[t, key, v]);
        }
        other => {
            let tag = match other {
                RegKind::Float => raw::FLOAT,
                RegKind::Table => raw::TABLE,
                RegKind::Closure => raw::CLOSURE,
                RegKind::Str => raw::STR,
                RegKind::Nil | RegKind::Int | RegKind::Unset => unreachable!(),
            };
            let v = bcx.use_var(val_var);
            let tag_v = bcx.ins().iconst(types::I64, tag as i64);
            let f = module.declare_func_in_func(set_raw_id, bcx.func);
            bcx.ins().call(f, &[t, key, v, tag_v]);
        }
    }
}

/// P14-S14-B v4 — flush context for the buffered string
/// accumulator emit. When `Some`, both
/// `emit_store_back_and_return_*` emit a `luna_jit_str_buf_intern`
/// + `def_var(accum_slot, str_ptr)` + `luna_jit_str_buf_release`
/// sequence BEFORE the existing store-back loop, so the
/// accumulator slot holds a real LuaStr ptr by the time the
/// dispatcher restores from reg_state.
#[derive(Clone, Copy)]
struct FlushCtx {
    buf_var: Variable,
    accum_slot: u32,
    intern_ref: cranelift_codegen::ir::FuncRef,
    release_ref: cranelift_codegen::ir::FuncRef,
}

/// v2.0 Track-R R3.3+ sub-1 — depth-relative base address helper.
///
/// Given the trace's `base_var` Variable (declared at entry block, see
/// `lower_trace_into_named` sub-1 scaffold) plus an op's window
/// offset (`op_offset_bytes` = `op_offsets[i] * 8`) plus a slot index
/// within that op's window, returns a `(base_value, byte_offset)`
/// pair suitable for `bcx.ins().load(..., base_value, byte_offset)`
/// or `bcx.ins().store(..., base_value, byte_offset)`.
///
/// Sub-1 caller contract: `base_var` is initialised to `iconst(0)` —
/// i.e., a depth-0 sentinel placeholder. Calling this helper produces
/// load/store IR that addresses `[0 + op_offset_bytes + slot * 8]`,
/// which is NOT a valid reg_state-relative address. Sub-1 op-arms
/// MUST NOT call this helper (they keep using `regs_full[off + slot]`
/// via `bcx.use_var` / `bcx.def_var`). The helper exists only as the
/// threading-shape proof: sub-2 will (a) replace the iconst(0) init
/// with `reg_state` itself + (b) start migrating Op::Move / Op::LoadK
/// / Op::LoadNil arms to call this helper instead of indexing
/// `regs_full`. Sub-3 will insert the R3d stitch_blk base-shift
/// `iadd_imm(base_var, -8 * recorded_delta)` BETWEEN the cmp brif and
/// the store-back so the deopt path lands at the caller window
/// (Risk D1.R2 mitigation in `.dev/rfcs/v2.0-track-r-r3-3-rfc.md` §8).
///
/// Why a helper (not inline `iadd_imm` at each call site): the
/// op_offset + slot arithmetic is identical across all op-arms and
/// the LJ source citation in `.dev/rfcs/v2.0-track-r-r3-3-luajit-
/// study.md` shows arm64's `ldr Xd, [Xn, #imm]` handles the pattern
/// in a single addressing mode. Concentrating the math in one helper
/// keeps the Cranelift mid-end's `iadd_imm` coalescing surface
/// uniform (Risk D1.R1 mitigation) and lets sub-2 audit codegen at
/// ONE site instead of ~30.
// Sub-1 scaffold: no production caller yet (sub-2 will migrate op-arms
// to call this; sub-1's only consumer is the regression test smoke
// probe at `r3_3_sub1_base_var_scaffold.rs`). `pub(crate)` so the
// test crate's hook can dispatch through `try_compile_trace_with_options`
// — the helper itself stays internal because sub-2 is the place where
// op-arm rewiring decides the final visibility.
#[allow(dead_code)]
pub(crate) fn current_base_addr(
    bcx: &mut FunctionBuilder<'_>,
    base_var: Variable,
    op_offset_bytes: i32,
    slot: u32,
) -> (Value, i32) {
    let base_now = bcx.use_var(base_var);
    let total_offset = op_offset_bytes.saturating_add((slot as i32).saturating_mul(8));
    (base_now, total_offset)
}

fn emit_flush_buf(bcx: &mut FunctionBuilder<'_>, ctx: &FlushCtx, regs: &[Variable]) {
    let buf_ptr = bcx.use_var(ctx.buf_var);
    let call_inst = bcx.ins().call(ctx.intern_ref, &[buf_ptr]);
    let str_ptr = bcx.inst_results(call_inst)[0];
    if let Some(&accum_var) = regs.get(ctx.accum_slot as usize) {
        bcx.def_var(accum_var, str_ptr);
    }
    bcx.ins().call(ctx.release_ref, &[buf_ptr]);
}

/// P15-A v2-C-A2 — emit the indirect-call-or-return gate. Loads
/// the cell at `side_trace_cell_addr`; if non-null, tail-calls the
/// child fn via `call_indirect`, ORs sentinel bits 56..=63 into the
/// child's i64 return, and returns. Otherwise runs the caller's
/// `normal_return` closure. The caller must have ALREADY written the
/// reg_state slots before calling this (the child reads them via its
/// entry block).
fn emit_side_trace_or_return(
    bcx: &mut FunctionBuilder<'_>,
    reg_state: Value,
    side_trace_cell_addr: i64,
    trace_fn_sig_ref: cranelift_codegen::ir::SigRef,
    sentinel_code: u32,
    normal_return: impl FnOnce(&mut FunctionBuilder<'_>),
) {
    // P15-A v2-C-A7 — `side_trace_cell_addr == 0` is the "no-gate"
    // sentinel: emit the normal return only, skipping the load +
    // icmp + brif + call_indirect IR. A6 mini N=3 showed the gate
    // is a NET PERF LOSS when it fires at every dispatch site (19
    // callsites × `load + icmp + brif` per parent dispatch >
    // amortization from rare side-trace fires). A7 restricts the
    // gate to TAG callsites (3) where hot exits actually live; the
    // 14 GLOBAL + 2 INLINE callsites pass 0 here and avoid the
    // overhead. The close-handler still writes child entry ptrs to
    // the legacy `exit_side_trace_ptrs` + the per-kind cells so the
    // counters stay populated; only the IR gate emission is gated.
    if side_trace_cell_addr == 0 {
        normal_return(bcx);
        return;
    }
    let cell_addr = bcx.ins().iconst(types::I64, side_trace_cell_addr);
    let fn_ptr = bcx
        .ins()
        .load(types::I64, MemFlags::trusted(), cell_addr, 0);
    let null = bcx.ins().iconst(types::I64, 0);
    let has_side = bcx.ins().icmp(IntCC::NotEqual, fn_ptr, null);
    let do_side_blk = bcx.create_block();
    let do_exit_blk = bcx.create_block();
    bcx.ins().brif(has_side, do_side_blk, &[], do_exit_blk, &[]);

    // Side block: indirect call into the child trace. ABI matches
    // the parent's own (`(I64) -> I64`); pass the same reg_state
    // pointer so the child sees the just-stored slots. OR sentinel
    // bits 56..=63 into the child's i64 return so the dispatcher
    // re-decodes via the SIDE TRACE's shape inputs.
    bcx.switch_to_block(do_side_blk);
    bcx.seal_block(do_side_blk);
    let call_inst = bcx
        .ins()
        .call_indirect(trace_fn_sig_ref, fn_ptr, &[reg_state]);
    let body = bcx.inst_results(call_inst)[0];
    let mask_u64: u64 = (1u64 << 63) | ((sentinel_code as u64 & 0x7F) << 56);
    let mask_v = bcx.ins().iconst(types::I64, mask_u64 as i64);
    let masked = bcx.ins().bor(body, mask_v);
    bcx.ins().return_(&[masked]);

    // Exit block: caller-provided normal return path (the existing
    // pre-v2-C encoded-return semantics).
    bcx.switch_to_block(do_exit_blk);
    bcx.seal_block(do_exit_blk);
    normal_return(bcx);
}

fn emit_store_back_and_return_pc(
    bcx: &mut FunctionBuilder<'_>,
    regs: &[Variable],
    reg_state: Value,
    pc: u32,
    flush_ctx: Option<&FlushCtx>,
    side_trace_cell_addr: i64,
    trace_fn_sig_ref: cranelift_codegen::ir::SigRef,
    sentinel_code: u32,
) {
    if let Some(ctx) = flush_ctx {
        emit_flush_buf(bcx, ctx, regs);
    }
    for (idx, v) in regs.iter().copied().enumerate() {
        let val = bcx.use_var(v);
        let offset = (idx as i32) * 8;
        bcx.ins().store(MemFlags::new(), val, reg_state, offset);
    }
    emit_side_trace_or_return(
        bcx,
        reg_state,
        side_trace_cell_addr,
        trace_fn_sig_ref,
        sentinel_code,
        |bcx| {
            let pc_val = bcx.ins().iconst(types::I64, pc as i64);
            bcx.ins().return_(&[pc_val]);
        },
    );
}

/// P12-S4-step4b-C-2 — inline cmp@d>0 side-exit return shape. The
/// upper 32 bits encode `site_idx + 1` (1-based; 0 means "no
/// inline site, look up via cont_pc in `per_exit_tags`"); the lower
/// 32 bits hold the resume PC. The dispatcher decodes this so a
/// cont_pc shared across multiple inline cmps (fib has 4+ such
/// sites colliding on pc=3) maps to the right entry's exit_tags
/// and chain.
fn emit_store_back_and_return_site(
    bcx: &mut FunctionBuilder<'_>,
    regs: &[Variable],
    reg_state: Value,
    site_idx: u32,
    cont_pc: u32,
    flush_ctx: Option<&FlushCtx>,
    side_trace_cell_addr: i64,
    trace_fn_sig_ref: cranelift_codegen::ir::SigRef,
) {
    if let Some(ctx) = flush_ctx {
        emit_flush_buf(bcx, ctx, regs);
    }
    for (idx, v) in regs.iter().copied().enumerate() {
        let val = bcx.use_var(v);
        let offset = (idx as i32) * 8;
        bcx.ins().store(MemFlags::new(), val, reg_state, offset);
    }
    let sentinel = encode_side_sentinel(SIDE_SENT_KIND_INLINE, site_idx);
    emit_side_trace_or_return(
        bcx,
        reg_state,
        side_trace_cell_addr,
        trace_fn_sig_ref,
        sentinel,
        |bcx| {
            let encoded = (((site_idx as u64) + 1) << 32) | (cont_pc as u64);
            let v = bcx.ins().iconst(types::I64, encoded as i64);
            bcx.ins().return_(&[v]);
        },
    );
}

/// P12-S2.B step 5 — lowerer for Int arith + Move + Int-Int cmp
/// guards + Table ops + trace-truncating `Op::Call` on a
/// single-Proto trace.
///
/// Attempt to lower a closed [`TraceRecord`] to a native trace fn.
/// The fn's ABI is `fn(reg_state: *mut i64) -> i64` (see [`TraceFn`]).
/// At entry, the trace loads every register from the caller's
/// `reg_state` buffer into a cranelift `Variable`; the body emits
/// IR per op. Each `Lt / Le / Eq` op emits an `icmp` + `brif` —
/// on a runtime mismatch with the recorded comparison direction,
/// control diverts to a side-exit block that stores reg state back
/// and returns the failing PC. Each `NewTable / SetI / GetI / Len`
/// op emits a cranelift `call` to the matching `luna_jit_*` helper
/// (`Linkage::Import`, resolved via `JITBuilder::symbol`); helpers
/// short-circuit on `vm.jit.pending_err` so a metatable-bearing
/// table parks a deopt request the dispatcher (S2.C / S3) can
/// detect after the trace returns. The clean-close tail stores
/// reg state back and returns `head_pc as i64`.
///
/// Returns `None` if:
/// - the record is not closed yet (open traces can't be entered
///   safely — the loop edge is the only sound entry/exit),
/// - any recorded op is outside `is_whitelisted_step4`,
/// - any recorded op comes from a Proto other than `head_proto`
///   (inlined sub-calls don't ship until S4),
/// - any recorded op has `inline_depth > 0` (same reason),
/// - any operand register index ≥ `head_proto.max_stack`,
/// - a `Lt / Le / Eq` is not followed by a `Jmp` at `cmp.pc + 1`
///   (the only direction step 3 captures),
/// - a `Jmp` is neither cmp-consumed nor at the trace's last
///   position,
/// - cranelift codegen fails.
///
/// On success, the underlying `JITModule` is parked on
/// `storage.trace_handles` so the returned `CompiledTrace.entry`
/// stays callable for the lifetime of the owning `Vm`.
///
/// **Caller contract for table ops** (step 4): before invoking the
/// returned entry, the caller (a test harness today; the S3
/// dispatcher tomorrow) must call [`crate::jit_backend::enter_jit`] to pin
/// the active Vm in the `JIT_VM` thread-local — the table helpers
/// pick that up to reach `vm.heap`. After the call, the caller
/// must inspect `vm.jit.pending_err` to decide whether a metatable
/// deopt fired; on `Some`, treat the trace's result as invalid and
/// re-run the work through the interpreter.
///
/// **Still no Vm::run caller** — `Vm::run` does not invoke this in
/// step 4. Behavior change to the interpreter / benchmarks: none.
/// S2.C will wire `try_compile_trace` into the close handler.
///
/// This is a convenience wrapper for callers that don't need to
/// pick options — it forwards to
/// [`try_compile_trace_with_options`] with [`CompileOptions::default`]
/// (one-shot, the shape unit tests assume).
pub fn try_compile_trace(
    storage: &mut dyn luna_core::jit::JitStorage,
    record: &TraceRecord,
) -> Option<CompiledTrace> {
    try_compile_trace_with_options(storage, record, CompileOptions::default())
}

// P13-S13-G v2.6 — last-checkpoint instrumentation for trace
// compile failure diagnosis. `try_compile_trace_with_options`
// updates the thread-local at each major phase; if the function
// returns `None`, the most recent checkpoint set tells the
// caller WHICH phase bailed. Vm reads + accumulates this on
// every compile-failed return.
thread_local! {
    pub(crate) static LAST_COMPILE_CHECKPOINT: std::cell::Cell<&'static str> =
        const { std::cell::Cell::new("not-entered") };
    pub(crate) static LAST_OP_ID: std::cell::Cell<u8> =
        const { std::cell::Cell::new(255) };
    // v2.0 Track-R R3.3+ sub-1 — counter bumped exactly once per
    // `lower_trace_into_named` invocation that successfully declares
    // the depth-relative `base_var` scaffold. Used by the sub-1
    // regression test (`r3_3_sub1_base_var_scaffold.rs`) to assert
    // the scaffold's declaration ran end-to-end without actually
    // exercising any op-arm migration (sub-2 territory).
    //
    // Probe-only: dispatched + close-cause counters cover production
    // behaviour; this cell exists solely so the test can pin "scaffold
    // ran" without scraping Cranelift IR text. The bump happens AFTER
    // `declare_var` + `def_var(iconst(0))` so a panic earlier in the
    // entry block leaves the counter at its prior value.
    pub(crate) static BASE_VAR_SCAFFOLD_DECLARED: std::cell::Cell<u64> =
        const { std::cell::Cell::new(0) };
}

fn checkpoint(s: &'static str) {
    LAST_COMPILE_CHECKPOINT.with(|c| c.set(s));
}

fn set_last_op_id(id: u8) {
    LAST_OP_ID.with(|c| c.set(id));
}

/// Name of the lowerer checkpoint most recently reached on this thread.
/// Diagnostic-only — used to bucket trace-compile failures by phase
/// (`pre-lower`, `lower-loop`, `finalize`, …).
pub fn last_compile_checkpoint() -> &'static str {
    LAST_COMPILE_CHECKPOINT.with(|c| c.get())
}

/// Opcode id (luna `Op` discriminant) of the last bytecode op the
/// lowerer touched on this thread. Diagnostic-only.
pub fn last_op_id() -> u8 {
    LAST_OP_ID.with(|c| c.get())
}

/// v2.0 Track-R R3.3+ sub-1 — count of successful `base_var` scaffold
/// declarations on this thread. Bumped exactly once per
/// `lower_trace_into_named` invocation that reaches the post-entry
/// emit point and runs `declare_var` + `def_var(iconst(0))` for the
/// depth-relative base address handle. Sub-1 scaffold-only: NO op-arm
/// migration (sub-2 territory); the Variable is in-scope for the
/// entire lowerer body but `use_var(base_var)` doesn't happen yet.
///
/// Read by `r3_3_sub1_base_var_scaffold.rs`; production paths
/// (dispatcher / close handler / vm) never read this.
///
/// See `.dev/rfcs/v2.0-track-r-r3-3-sub1-verdict.md` for the scaffold
/// shape decision (single-def_var, no audit anchor) + sub-2 handoff
/// (replace `iconst(0)` init with `reg_state` + migrate Op::Move /
/// Op::LoadK / Op::LoadNil arms to call `current_base_addr`).
pub fn base_var_scaffold_declared_count() -> u64 {
    BASE_VAR_SCAFFOLD_DECLARED.with(|c| c.get())
}

/// v2.0 Track-R R3.3+ sub-1 — reset the scaffold-declared counter so
/// a regression test can assert "the next compile bumped it by 1"
/// without depending on prior tests in the same thread. Test-only;
/// production paths never call this.
pub fn reset_base_var_scaffold_declared_count() {
    BASE_VAR_SCAFFOLD_DECLARED.with(|c| c.set(0));
}

/// v1.3 Phase AOT Stage 3 — build a fresh `JITModule` configured with
/// every trace-side `luna_jit_*` helper symbol registered for
/// `Linkage::Import` resolution at finalize time.
///
/// Companion of [`super::build_jit_module_with_helpers`] (the int-chunk
/// counterpart). The AOT pipeline (luna-aot) builds an `ObjectModule`
/// instead and resolves the same symbols at static-link time — see
/// `.dev/rfcs/v1.3-audit-luna-aot.md` § "Stage 3: bytecode → Cranelift
/// IR (the shared lowerer)".
///
/// **Stage 3 status**: both the int-chunk lowerer
/// ([`super::lower_int_chunk_into`]) and the trace lowerer
/// ([`lower_trace_into`]) are fully generic over
/// `M: cranelift_module::Module`. The two emit-time helper free fns
/// ([`emit_table_set`] / [`emit_materialize_live_sunk`]) are also
/// generic. JIT-specific surfaces remaining are: this module-
/// construction helper, the JIT wrappers ([`try_compile_trace_with_options`]
/// + [`super::try_compile_int_chunk`]) that finalize, and the
/// `TraceHandle` / `JitHandle` types that own the mmap'd `JITModule`
/// for the entry's lifetime. The trace lowerer returns a
/// `CompiledTrace` with [`placeholder_trace_fn`] in `entry`; the JIT
/// wrapper patches the real fn pointer after `finalize_definitions` +
/// `get_finalized_function`. The AOT pipeline (luna-aot) never
/// invokes the entry directly — it resolves the trace symbol at
/// static-link time and dispatches through its own table.
fn build_trace_jit_module() -> Option<JITModule> {
    let mut flag_builder = settings::builder();
    flag_builder.set("use_colocated_libcalls", "false").ok()?;
    flag_builder.set("is_pic", "false").ok()?;
    flag_builder.set("opt_level", "speed").ok()?;
    let isa = cranelift_native::builder()
        .ok()?
        .finish(settings::Flags::new(flag_builder))
        .ok()?;
    let mut builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
    // Step 4 emits `Op::NewTable / SetI / GetI / Len` as calls to
    // the method JIT's `luna_jit_*` helpers — register the symbols
    // so cranelift's `Linkage::Import` resolver finds them at
    // finalize time. (rlib link strips `#[no_mangle]` for executables
    // like `cargo test`, so the default `dlsym(RTLD_DEFAULT)` resolver
    // misses them without an explicit `builder.symbol(...)`.)
    builder.symbol("luna_jit_new_table", super::luna_jit_new_table as *const u8);
    builder.symbol(
        "luna_jit_table_set_int",
        super::luna_jit_table_set_int as *const u8,
    );
    // P12-S6-A2 — Nil-valued SetList/SetI/SetTable helper. Trace JIT
    // emits a call here when an Op::LoadNil-written source register
    // is fed into a (non-sunk) table write.
    builder.symbol(
        "luna_jit_table_set_nil",
        super::luna_jit_table_set_nil as *const u8,
    );
    // P12-S7-C — generalised SetTable/SetI/SetList helper for any
    // (tag, raw_bits) pair. Used for Closure / Table / non-Int/Nil
    // sources where the legacy set_int helper would mis-wrap as
    // Value::Int(ptr_bits).
    builder.symbol(
        "luna_jit_table_set_raw",
        super::luna_jit_table_set_raw as *const u8,
    );
    // P12-S11-A — SetField + GetField helpers (string key from
    // Proto.consts; raw pointer baked into IR at emit time).
    builder.symbol(
        "luna_jit_table_set_field",
        super::luna_jit_table_set_field as *const u8,
    );
    builder.symbol(
        "luna_jit_table_get_field",
        super::luna_jit_table_get_field as *const u8,
    );
    // v1.2 D3 Path B — standalone GetTabUp helper.
    builder.symbol(
        "luna_jit_op_get_tab_up",
        super::luna_jit_op_get_tab_up as *const u8,
    );
    builder.symbol(
        "luna_jit_table_get_int",
        super::luna_jit_table_get_int as *const u8,
    );
    builder.symbol("luna_jit_table_len", super::luna_jit_table_len as *const u8);
    // P12-S4-step2b — `Op::GetUpval` reads `cl.upvals[idx]` via this
    // helper. Reuses the method JIT helper; the trace dispatcher's
    // `enter_jit(vm, Some(cl))` pins `JIT_CL` so the helper can find
    // the closure at runtime.
    builder.symbol("luna_jit_upval_get", super::luna_jit_upval_get as *const u8);
    // P12-S4-step4b-A — frame materialization helper. Step4b-C will
    // emit calls to it from the cmp@d>0 side-exit path. Register the
    // symbol unconditionally so the lowerer can declare the import
    // without needing per-trace gating; cranelift's dead-symbol
    // elimination drops the import if no IR references it.
    builder.symbol(
        "luna_jit_trace_materialize_frames",
        super::luna_jit_trace_materialize_frames as *const u8,
    );
    // P12-S5-C — sunk-table materialise helper. Emit calls it at
    // each cmp side-exit (depth=0 today) for every live Sinkable
    // site whose virt slots must reach interp via the heap path.
    builder.symbol(
        "luna_jit_materialize_sunk_table",
        super::luna_jit_materialize_sunk_table as *const u8,
    );
    // P12-S7-A — Op::Closure shared-upval helper.
    builder.symbol(
        "luna_jit_op_closure",
        super::luna_jit_op_closure as *const u8,
    );
    // P12-S7-B — pre-Closure spill helper. Emit calls this once
    // per in_stack upval just before luna_jit_op_closure so
    // find_or_create_upval sees a live vm.stack slot.
    builder.symbol(
        "luna_jit_spill_to_stack",
        super::luna_jit_spill_to_stack as *const u8,
    );
    // P12-S7-C — Op::Close predict-and-deopt helper.
    builder.symbol("luna_jit_op_close", super::luna_jit_op_close as *const u8);
    // P12-S12-B-v2 — generic-for helpers. `op_tforcall` runs the
    // iterator function via vm.begin_call (Native iters only — Lua
    // closure iters deopt); `stack_load` / `stack_tag` read vm.stack
    // back into trace IR `Variable`s after the helper has mutated
    // R[A+2] (control) and R[A+4..] (returned key/value).
    builder.symbol(
        "luna_jit_op_tforcall",
        super::luna_jit_op_tforcall as *const u8,
    );
    builder.symbol(
        "luna_jit_stack_load",
        super::luna_jit_stack_load as *const u8,
    );
    builder.symbol("luna_jit_stack_tag", super::luna_jit_stack_tag as *const u8);
    // P12-S12-C v1 — Op::Concat helpers.
    builder.symbol("luna_jit_op_concat", super::luna_jit_op_concat as *const u8);
    builder.symbol(
        "luna_jit_stack_update_raw",
        super::luna_jit_stack_update_raw as *const u8,
    );
    // P14-S14-B v2 — string accumulator buffer pool helpers.
    builder.symbol(
        "luna_jit_str_buf_acquire",
        super::luna_jit_str_buf_acquire as *const u8,
    );
    builder.symbol(
        "luna_jit_str_buf_release",
        super::luna_jit_str_buf_release as *const u8,
    );
    builder.symbol(
        "luna_jit_str_buf_extend",
        super::luna_jit_str_buf_extend as *const u8,
    );
    builder.symbol(
        "luna_jit_str_buf_intern",
        super::luna_jit_str_buf_intern as *const u8,
    );
    Some(JITModule::new(builder))
}

/// v1.3 Phase AOT Stage 3 placeholder `TraceFn` — installed in
/// [`CompiledTrace::entry`] by the backend-agnostic [`lower_trace_into`]
/// body and patched to the real finalized entry pointer by the JIT
/// wrapper [`try_compile_trace_with_options`]. The AOT pipeline
/// (`luna-aot`) never calls into a TraceFn directly (resolution happens
/// at the deploy-side runtime, not at codegen time), so the placeholder
/// reaching the AOT output is harmless; the deploy-side runtime
/// resolves the trace symbol through the static dispatch table built in
/// `luna-aot`'s embed pipeline.
unsafe extern "C" fn placeholder_trace_fn(_reg_state: *mut i64) -> i64 {
    panic!(
        "placeholder_trace_fn called: CompiledTrace.entry must be patched by the JIT finalize wrapper before dispatch"
    );
}

/// Variant of [`try_compile_trace`] that takes a [`CompileOptions`]
/// — the close handler uses this with `internal_loop = true` so the
/// JIT'd trace runs in a native loop until a cmp side-exits.
///
/// v1.3 Phase AOT Stage 3 — thin wrapper around the backend-agnostic
/// [`lower_trace_into`] generic. Constructs a `JITModule`, finalizes
/// the compiled trace into RWX memory, patches the real entry fn ptr
/// into the returned [`CompiledTrace`], and stashes the module in
/// `storage.trace_handles` so the entry stays callable for the
/// lifetime of the owning `Vm`.
pub fn try_compile_trace_with_options(
    storage: &mut dyn luna_core::jit::JitStorage,
    record: &TraceRecord,
    opts: CompileOptions,
) -> Option<CompiledTrace> {
    let mut module = build_trace_jit_module()?;
    let (fn_id, mut compiled) = lower_trace_into(&mut module, record, opts)?;
    module.finalize_definitions().ok()?;
    let ptr = module.get_finalized_function(fn_id);
    // SAFETY: the cranelift fn signature declared by `lower_trace_into`
    // (`(I64) -> I64`) matches `TraceFn`. The mmap backing the fn body
    // is owned by `module`, which we park on the per-`Vm` storage's
    // `trace_handles` Vec immediately below.
    // v2.0 Track J sub-step J-B Phase F — was `TRACE_JIT_HANDLES` TLS;
    // now per-`Vm` field on storage.
    let entry_fn: TraceFn = unsafe { std::mem::transmute::<*const u8, TraceFn>(ptr) };
    compiled.entry = entry_fn;
    // v2.0 J-B follow-up — `from_storage` is `Result`-shaped now. On
    // `StorageMismatch` (Vm.jit.storage isn't a CraneliftJitStorage)
    // skip parking the handle and return `None` — the freshly built
    // `module` drops here and releases its mmap pages; the trace
    // recorder sees `None` and gives up on this trace, falling back
    // to interp dispatch. No SIGABRT across the C-ABI boundary.
    let cs = crate::jit_backend::storage::from_storage(storage).ok()?;
    cs.trace_handles.push(TraceHandle {
        // v2.0 Track J sub-step J-D — wrap in `SendJitModule`
        // sleeve. SAFETY: `build_trace_jit_module` uses the
        // default `SystemMemoryProvider` path (no
        // `JITBuilder::memory_provider` call).
        _module: super::SendJitModule::new(module),
        _entry_raw: ptr,
    });
    Some(compiled)
}

/// v1.3 Phase AOT Stage 3 — backend-agnostic body of the trace
/// lowerer. Generic over any `cranelift_module::Module` so the same
/// codegen pipeline drives the runtime JIT (`JITModule`,
/// [`try_compile_trace_with_options`]) and the AOT pipeline
/// (`ObjectModule` in `luna-aot`).
///
/// Returns `None` on the same bail conditions as
/// [`try_compile_trace`] (see its docstring). On success returns the
/// declared [`FuncId`] for the lowered trace alongside a
/// [`CompiledTrace`] whose `entry` field holds a private
/// `placeholder_trace_fn`; backend-specific finalize must patch the
/// real entry pointer before dispatch (the JIT wrapper does this; the
/// AOT pipeline resolves the symbol at link time and never invokes
/// `entry` directly).
pub fn lower_trace_into<M: Module>(
    module: &mut M,
    record: &TraceRecord,
    opts: CompileOptions,
) -> Option<(FuncId, CompiledTrace)> {
    lower_trace_into_named(module, record, opts, None)
}

/// v1.3 Phase AOT Stage 7 sub-piece 4 — like [`lower_trace_into`] but
/// lets the caller (luna-aot) pick a unique exported name for the
/// trace function. Required for AOT: many traces from the same chunk
/// would otherwise collide on `"luna_jit_trace"`, and `Linkage::Local`
/// hides the symbol from the deploy-side staticlib's resolver.
///
/// `aot_fn_name = None` keeps the original behaviour (anonymous
/// `Linkage::Local` `"luna_jit_trace"`), so the JIT wrapper and the
/// existing AOT smoke tests are unaffected.
///
/// When `Some(name)`, `name` becomes the cranelift `FuncId` symbol
/// with `Linkage::Export`, surfacing in the produced `.o`'s symbol
/// table for the deploy-side `dlsym`/linker to resolve.
pub fn lower_trace_into_named<M: Module>(
    mut module: &mut M,
    record: &TraceRecord,
    opts: CompileOptions,
    aot_fn_name: Option<&str>,
) -> Option<(FuncId, CompiledTrace)> {
    checkpoint("enter");
    if !record.closed {
        checkpoint("bail:not-closed");
        return None;
    }
    checkpoint("post:closed-check");

    // v1.3 Phase AOT Stage 7 sub-piece 2 — track which AOT data slots
    // we've already `define_data`'d this lower call. `declare_data`
    // returns the same `DataId` for the same name (Cranelift name
    // interning), but `define_data` rejects redefinition with
    // `ModuleError::DuplicateDefinition` — so the dedupe guard sits
    // around `define_data`, not `declare_data`.
    let mut defined_aot_data: std::collections::HashSet<DataId> = std::collections::HashSet::new();

    let head_proto = record.head_proto;
    let max_stack = head_proto.max_stack as usize;
    let n = record.ops.len();

    // P12-S4-step3b — recorder invariant: the first recorded op is at
    // depth 0 on `head_proto`. A record violating either would break
    // `compute_op_offsets`' depth-bump arithmetic; bail cleanly here
    // rather than panic deeper in.
    if let Some(first) = record.ops.first() {
        if first.inline_depth != 0 || !std::ptr::eq(first.proto.as_ptr(), head_proto.as_ptr()) {
            checkpoint("bail:first-op-shape");
            return None;
        }
    }
    checkpoint("post:first-op-check");

    // P15-A v2-E smart side-trace gate is moved BELOW
    // `compute_op_offsets` so it can reuse the verified op_offsets
    // (calling compute_op_offsets early can panic if the depth
    // invariant fails — verify_depth_invariant runs at line ~3444).

    // P12-S4-step3b — per-op register-window offsets across inlined
    // self-recursive frames. `op_offsets[i]` is the start of op i's
    // register window inside reg_state_buf; `enclosing_call_a[i]` is
    // the matching caller `Op::Call`'s A field (None at depth 0).
    // `window_size` is the largest `off + max_stack` across all ops —
    // sized so even the deepest inlined frame fits. The dispatcher
    // (vm/exec.rs) reads `window_size` off `CompiledTrace` to size
    // its reg_state buffer; only [0..max_stack) is marshalled in
    // from the interp stack, [max_stack..window_size) is zero-init
    // and filled by the trace's own GetUpval / arith.
    // P13-S13-A — consolidated depth invariant check. Bails if
    // any of:
    //   - first op not at depth 0 (already checked above against
    //     head_proto, but kept here for the pure-function test)
    //   - any consecutive ops jump > 1 depth (e.g. Op::Close
    //     pushing both a Cont::Close frame AND a handler's Lua
    //     frame — recorder sees 0 → 2, IR has no intermediate)
    //   - a depth bump is not preceded by an Op::Call (recorder
    //     contract: only Op::Call can push a new frame)
    //   - any op exceeds MAX_INLINE_DEPTH (the lowerer caps its
    //     window_size on this)
    // The check is pulled into `verify_depth_invariant` (lib
    // unit tested over synthetic depth/Op-is-Call sequences;
    // doesn't need a real `Gc<Proto>`).
    let depth_items: Vec<(u8, bool)> = record
        .ops
        .iter()
        .map(|r| (r.inline_depth, matches!(r.inst.op(), Op::Call)))
        .collect();
    if !verify_depth_invariant(&depth_items) {
        checkpoint("bail:depth-invariant");
        return None;
    }
    checkpoint("post:depth-invariant");
    let (op_offsets, enclosing_call_a) = compute_op_offsets(record);
    let mut window_size: u32 = op_offsets
        .iter()
        .map(|&off| off + max_stack as u32)
        .max()
        .unwrap_or(max_stack as u32);
    // P16-B — SelfLink close needs `regs_full` to extend through the
    // would-be-next-depth's window so the snapshot-restore copy reads
    // from valid slots. Without this extension, compute_op_offsets
    // only covers the deepest CAPTURED depth (the recorder closed
    // BEFORE pushing the tripping depth's frame), and bump-target
    // reads would go OOB. Extend by one max_stack window past the
    // last Op::Call's bump destination.
    if record.self_link_kind.is_some() {
        let mut last_call_idx: Option<usize> = None;
        for (i, rop) in record.ops.iter().enumerate() {
            if matches!(rop.inst.op(), Op::Call) {
                last_call_idx = Some(i);
            }
        }
        if let Some(idx) = last_call_idx {
            let bump_off = op_offsets[idx] + record.ops[idx].inst.a() + 1;
            let needed = bump_off + max_stack as u32;
            if needed > window_size {
                window_size = needed;
            }
        }
    }
    let window_size_us = window_size as usize;

    // P15-A v2-E — SMART side-trace gate (replaces the v2-C-A6-5
    // back-edge bail). Compute the child's read-before-write live-
    // in slot set (slots READ without first being WRITTEN within
    // child's body — values carried in from the parent's exit
    // reg_state). Intersect with the parent's body_writes (slots
    // the parent's recorded body writes — values that go STALE
    // across child's internal-loop iters because parent doesn't
    // re-run those writes mid-side-trace). Non-empty intersection
    // = the s12_step_b class of bug; bail compile. Empty = side
    // trace is self-contained w.r.t. parent's writes — safe to
    // internal-loop OR forward-only — allow either.
    //
    // More permissive than the v2-C-A6-5 back-edge bail (which
    // banned ALL back-edge ops in side traces): self-contained
    // back-edge side traces (e.g. recursive call branches that
    // re-compute their inputs each iter) can now compile and
    // amortize the parent's hot-exit dispatch cost.
    if let Some((parent_proto, parent_head_pc, _)) = record.side_trace_parent {
        // Check 1: any back-edge op? (ForLoop / TForLoop / Jmp -bx)
        let has_back_edge = record.ops.iter().any(|op| match op.inst.op() {
            luna_core::vm::isa::Op::ForLoop | luna_core::vm::isa::Op::TForLoop => true,
            luna_core::vm::isa::Op::Jmp => op.inst.sbx() < 0,
            _ => false,
        });
        if has_back_edge {
            // Back-edge means the trace's IR will internal-loop OR
            // re-execute body ops. Two correctness requirements:
            //
            //   (a) child must not READ a slot the parent's body
            //       writes without first writing it itself (the
            //       s12_step_b stale-register bug).
            //   (b) child must not contain side-effect-producing
            //       ops (Call / TForCall / SetTable / SetI /
            //       SetField / SetUpval / SetTabUp / Closure /
            //       Close / Tbc) — these advance shared heap /
            //       iterator state that interp re-observes after
            //       the side trace returns, causing double-effect
            //       (the s12_step_d TForCall-double-advance bug).
            let has_impure = record.ops.iter().any(|op| {
                use luna_core::vm::isa::Op;
                matches!(
                    op.inst.op(),
                    Op::Call
                        | Op::TailCall
                        | Op::TForCall
                        | Op::SetTable
                        | Op::SetI
                        | Op::SetField
                        | Op::SetUpval
                        | Op::SetTabUp
                        | Op::Closure
                        | Op::Close
                        | Op::Tbc
                )
            });
            if has_impure {
                checkpoint("bail:side-trace-back-edge-with-impure");
                return None;
            }
            // Pure back-edge trace: still check live-in vs parent
            // writes (Add/Move loops can still re-read a stale
            // parent-written slot each iter).
            let child_live_in = compute_live_in_slots(record, &op_offsets);
            if !child_live_in.is_empty() {
                let parent_writes_opt = {
                    let traces = parent_proto.traces.borrow();
                    traces
                        .iter()
                        .find(|t| t.head_pc == parent_head_pc)
                        .map(|pct| pct.body_writes.clone())
                };
                if let Some(parent_writes) = parent_writes_opt {
                    let mut i = 0;
                    let mut j = 0;
                    let pw = &parent_writes[..];
                    let cl_li = &child_live_in[..];
                    while i < pw.len() && j < cl_li.len() {
                        match pw[i].cmp(&cl_li[j]) {
                            std::cmp::Ordering::Equal => {
                                checkpoint("bail:side-trace-live-in-overlap");
                                return None;
                            }
                            std::cmp::Ordering::Less => i += 1,
                            std::cmp::Ordering::Greater => j += 1,
                        }
                    }
                } else {
                    checkpoint("bail:side-trace-parent-ct-missing");
                    return None;
                }
            }
            // All back-edge checks passed; trace is allowed.
        }
        // No back-edge: forward-only is always safe (single-iter
        // execution; no internal-loop semantics to break).
    }
    checkpoint("post:side-trace-v2e-smart-gate");

    // P12-S4-step4b-C-1 — per-inlined-frame metadata for the
    // frame-mat helper. Walk record.ops; every self-recursive
    // Op::Call (next op at depth+1 on the same proto) describes one
    // callee frame the helper will push at side-exit time.
    //
    // Bail when:
    //   - any self-recursive Call has C != 2 (i.e. nresults != 1) —
    //     step3b's Op::Return1 copy-back assumes one return value
    //   - the head closure's proto is vararg — helper doesn't
    //     reconstruct the vararg rotation that `push_frame` does
    //
    // P12-S4-step4b-C-2 — frame-mat data is now per-cmp-site (the
    // RFC's "Lesson learned": single global indexed-by-depth array
    // gave the wrong chain to sibling-Call branches and looped fib
    // forever). Per-site `per_exit_metas` is built BELOW after
    // `cmp_dirs` are populated — that pass needs the cmp direction
    // to compute each site's side-exit PC.
    //
    // Pre-emit validation here: bail any self-recursive Call whose
    // `C != 2` (nresults != 1) — step3b's `Op::Return1` copy-back
    // assumes one return value and the materialize helper bakes
    // whatever the meta says without validating.
    for (i, rop) in record.ops.iter().enumerate() {
        if !matches!(rop.inst.op(), Op::Call) {
            continue;
        }
        let depth = rop.inline_depth as usize;
        let Some(next) = record.ops.get(i + 1) else {
            continue;
        };
        if (next.inline_depth as usize) != depth + 1 {
            continue;
        }
        if !std::ptr::eq(next.proto.as_ptr(), head_proto.as_ptr()) {
            continue;
        }
        // P12-S9-B — accept Call C=2 (single ret, original S4 path)
        // OR Call C=0 with var_count snapshot == 1 (multi-return
        // form that happens to return exactly 1 value, e.g.
        // binary_trees `make`'s `return {...}`). Both reduce to the
        // same emit (single-value Return1 copy-back from callee to
        // caller). For C=0 with var_count != 1, bail — multi-value
        // copy-back is S9-D.
        let c = rop.inst.c();
        if c == 2 {
            // OK, S4 path
        } else if c == 0 && rop.var_count == Some(1) {
            // OK, S9-B path (single return via variable form)
        } else {
            checkpoint("bail:self-rec-Call-c-not-1");
            return None;
        }
    }
    checkpoint("post:self-rec-Call-validate");
    // P12-S4-step4b-C-2 — also bail if the head proto is vararg.
    // The materialize helper builds frames with `n_varargs = 0`,
    // which doesn't reconstruct the vararg-rotated layout that
    // `push_frame` lays out for vararg functions. fib + simple
    // self-recursion isn't vararg; binary_trees TBD.
    if head_proto.is_vararg {
        for r in &record.ops {
            if r.inline_depth > 0 {
                checkpoint("bail:vararg-head-with-depth");
                return None;
            }
        }
    }
    checkpoint("post:vararg-check");

    // Find the first trace-terminating op. Two species:
    //
    // - `Op::Call` (step 5) *truncates* the trace: every op before
    //   gets normal IR, the Call emits a side-exit at its own PC,
    //   and every op after is dropped. Used as a placeholder until
    //   real Call inlining lands in S4.
    // - `Op::ForLoop` (step 6) is the numeric-for back-edge: every
    //   op before is the loop body, the ForLoop emits its own cmp
    //   + step + brif at the tail. Continue branch is the internal
    //   back-edge (or `return head_pc` in one-shot mode); side-exit
    //   branch returns `pc + 1` so interp resumes past the loop.
    //
    // Whichever appears *first* in the recorded ops takes the tail
    // slot — the other (if any) lives in the dropped region and is
    // ignored.
    // Scan for math folds first — a `Call` that's part of a fold
    // doesn't truncate the trace.
    let mut folded_ops: Vec<bool> = vec![false; n];
    let mut math_folds: Vec<TraceMathFold> = Vec::new();
    {
        // Recogniser does its own internal bounds check per arm
        // (libm1 needs 4 ops, min2/max2 needs ≥3 ops with the
        // Call no more than `MINMAX_FOLD_ARG_PREP_MAX` past
        // `start_idx + 1`). Walk every index; the matcher returns
        // None when the window doesn't fit so we don't run off
        // the end of `record.ops`.
        //
        // `folded_ops` bitmap layout:
        //   * `Libm1` — flags 4 consecutive indices
        //     `start_idx..=start_idx+3` (GetTabUp, GetField, Move,
        //     Call). All emit silently except `start_idx` which
        //     fires the libm call.
        //   * `Min2 / Max2` — flags only `start_idx`,
        //     `start_idx + 1`, and `call_idx`. Arg-prep ops in
        //     between are NOT folded — they execute normally so
        //     the Call args land at R[A+1] / R[A+2] by the
        //     standard Lua Call ABI. The Call's emit position
        //     fires `fmin / fmax`.
        let mut i = 0;
        while i < n {
            if let Some(fold) = try_match_trace_math_fold(record, i, head_proto) {
                match fold.kind {
                    FoldKind::Libm1 => {
                        for k in 0..4 {
                            folded_ops[i + k] = true;
                        }
                        i += 4;
                    }
                    FoldKind::Min2 | FoldKind::Max2 => {
                        folded_ops[fold.start_idx] = true;
                        folded_ops[fold.start_idx + 1] = true;
                        folded_ops[fold.call_idx] = true;
                        // Advance past the Call so the next scan
                        // starts beyond the recognised fold.
                        i = fold.call_idx + 1;
                    }
                }
                math_folds.push(fold);
            } else {
                i += 1;
            }
        }
    }

    // P12-S4-step3b — the terminator scan now also accounts for
    // inline self-recursion. Self-recursive Op::Call (next op is at
    // depth+1 on the same proto, within MAX_INLINE_DEPTH) is NOT a
    // terminator — body emit walks past it and op_offsets shifts the
    // register window for the callee.
    //
    // P12-S4-step4b-C-2 — cmp@d>0 NO LONGER closes via InlineAbort:
    // body emit calls the frame-mat helper at the side-exit then
    // returns side_exit_pc; dispatcher's restore loop walks the
    // newly-pushed inline frames. ForLoop@d>0 / non-self Call@d>0 /
    // depth past MAX_INLINE_DEPTH / proto mismatch still close via
    // InlineAbort (deferred to step5+).
    //
    // P12-S4-step4b-C-2 — Op::Return0/Return1 at depth=0 terminates
    // the trace via `TraceEnd::Return` (caller frame unwind). The
    // recorder closes the trace cleanly past the return; without
    // this truncation the Return would fail the whitelist check and
    // bail the whole compile.
    // v2.0 Track-R R3a — DownRec close wins over SelfLink: a depth>0
    // `Op::Return` re-trip of the recunroll threshold (captured at
    // `exec.rs` recorder gate) routes here BEFORE the SelfLink arm.
    // R3a only adds the variant + picker arm — the tail emit below
    // still falls through to the R1 safe `dispatchable=false` path
    // (the `self_link_idx_opt` arm) for now. R3b lifts that.
    //
    // R3b — effective_end for DownRec is the index of the natural
    // terminator (depth-0 Return/Call/ForLoop) so the body emit's
    // whitelist gate doesn't bail on the final op. R3a's effective_
    // end = `record.ops.len()` walked the natural close op as a body
    // op, which fails the `is_whitelisted_step5` check (Op::Return at
    // depth 0 is end-shape, not body) and aborted compile before the
    // DownRec tail arm could fire. R3b scans for the first natural
    // terminator and uses that idx (mirrors TraceEnd::Return / Call's
    // picker shape). Falls back to `record.ops.len()` only when no
    // natural terminator is present (R3a behaviour).
    let end_idx_opt: Option<(usize, TraceEnd)> = if let Some(dr) = record.downrec_close {
        let mut natural_end = record.ops.len();
        for (i, r) in record.ops.iter().enumerate() {
            if folded_ops[i] {
                continue;
            }
            let depth = r.inline_depth as usize;
            if depth != 0 {
                continue;
            }
            match r.inst.op() {
                Op::Call | Op::ForLoop | Op::TForLoop | Op::Return0 | Op::Return1 => {
                    natural_end = i;
                    break;
                }
                _ => {}
            }
        }
        Some((
            natural_end,
            TraceEnd::DownRec {
                return_pc: dr.return_pc,
                target_proto_id: dr.target_proto.as_ptr() as usize,
                depth_delta: dr.depth_delta,
            },
        ))
    } else if let Some(kind) = record.self_link_kind {
        // P16-A/B — self-link close overrides the natural terminator
        // scan. Recorder stopped capturing AT the head_pc re-entry
        // (about to re-execute the deepest-inlined frame's first op);
        // every prior op is intentional inline body. effective_end =
        // record.ops.len() so body emit walks the whole captured trail.
        // The tail emit picks the SelfLink arm (snapshot-restore +
        // branch-to-self) instead of any natural terminator that might
        // happen to sit at the end (e.g., a depth>0 Return in fib's
        // post-recursion add path that the recorder never actually
        // reaches in the cycle catch — but we guard against it anyway).
        Some((record.ops.len(), TraceEnd::SelfLink(kind)))
    } else {
        let mut found: Option<(usize, TraceEnd)> = None;
        for (i, r) in record.ops.iter().enumerate() {
            if folded_ops[i] {
                continue;
            }
            let depth = r.inline_depth as usize;
            if depth > MAX_INLINE_DEPTH as usize
                || !std::ptr::eq(r.proto.as_ptr(), head_proto.as_ptr())
            {
                found = Some((i, TraceEnd::InlineAbort));
                break;
            }
            match r.inst.op() {
                Op::Call => {
                    let nxt = record.ops.get(i + 1);
                    let is_self_recursive = nxt
                        .map(|n_op| {
                            n_op.inline_depth as usize == depth + 1
                                && depth < MAX_INLINE_DEPTH as usize
                                && std::ptr::eq(n_op.proto.as_ptr(), head_proto.as_ptr())
                        })
                        .unwrap_or(false);
                    if is_self_recursive {
                        // Continue walking — Op::Call emits nothing in
                        // the inline path and op_offsets handles the
                        // window shift for the callee's subsequent ops.
                        continue;
                    }
                    if depth == 0 {
                        found = Some((i, TraceEnd::Call));
                    } else {
                        found = Some((i, TraceEnd::InlineAbort));
                    }
                    break;
                }
                Op::ForLoop => {
                    if depth == 0 {
                        found = Some((i, TraceEnd::ForLoop));
                    } else {
                        found = Some((i, TraceEnd::InlineAbort));
                    }
                    break;
                }
                // P12-S12-B-v2 — generic-for back-edge. Same tail
                // emit slot as Op::ForLoop (TraceEnd::ForLoop); the
                // tail emit branches on `record.ops[idx].inst.op()`
                // to pick the right side-exit predicate (count>0 vs
                // R[A+4] tag check).
                Op::TForLoop => {
                    if depth == 0 {
                        found = Some((i, TraceEnd::ForLoop));
                    } else {
                        found = Some((i, TraceEnd::InlineAbort));
                    }
                    break;
                }
                Op::Return0 | Op::Return1 if depth == 0 => {
                    found = Some((i, TraceEnd::Return));
                    break;
                }
                // depth>0 Returns are inline-path unwinds; the
                // step3b emit loop handles them (Return0 no-op,
                // Return1 copy-back). Don't terminate.
                _ => {}
            }
        }
        found
    };
    let effective_end = end_idx_opt.map(|(i, _)| i).unwrap_or(n);
    // P12-S5-A/B — escape analysis over the recorded body +
    // terminator. S5-B's pre-emit pass below demotes any Sinkable
    // site that doesn't meet the v1 sunk-emit criteria back to
    // Escaped, so emit only honours sites we actually allocate
    // virt-slot Variables for.
    checkpoint("post:end-idx-found");
    let mut escape = escape_analyze(
        record,
        effective_end,
        end_idx_opt.map(|(_, k)| k),
        head_proto,
    );
    checkpoint("post:escape-analyze");

    // P14-S14-B v4-part2 — `flush_ctx` is declared mut here so
    // the entry-block setup below can populate it with
    // `Some(FlushCtx { ... })` when an active_accum is detected.
    // The 19 `emit_store_back_and_return_*` call sites all read
    // `flush_ctx.as_ref()`; the helpers no-op when it's None.
    let mut flush_ctx: Option<FlushCtx> = None;

    // P14-S14-B v4-part2 — detect the FIRST Bufferable AccumSite.
    // v4 ships single-site buffered emit. The 4 idiom op indices
    // are pre1 = op_idx-2, pre2 = op_idx-1, concat = op_idx,
    // post = op_idx+1.
    #[derive(Clone, Copy, Debug)]
    struct BufferedAccum {
        accum_slot: u32,
        piece_slot: u32,
        pre1_idx: usize,
        pre2_idx: usize,
        concat_idx: usize,
        post_idx: usize,
    }
    let active_accum: Option<BufferedAccum> = escape
        .accum_sites
        .iter()
        .find(|s| s.state == BufferState::Bufferable)
        .map(|s| BufferedAccum {
            accum_slot: s.accum_slot,
            piece_slot: s.piece_slot,
            pre1_idx: s.op_idx - 2,
            pre2_idx: s.op_idx - 1,
            concat_idx: s.op_idx,
            post_idx: s.op_idx + 1,
        });
    // Keep the old names working for the per-tail paths below.
    let call_idx_opt = match end_idx_opt {
        Some((i, TraceEnd::Call)) => Some(i),
        _ => None,
    };
    let for_loop_idx_opt = match end_idx_opt {
        Some((i, TraceEnd::ForLoop)) => Some(i),
        _ => None,
    };
    let inline_abort_idx_opt = match end_idx_opt {
        Some((i, TraceEnd::InlineAbort)) => Some(i),
        _ => None,
    };
    let return_idx_opt = match end_idx_opt {
        Some((i, TraceEnd::Return)) => Some(i),
        _ => None,
    };
    // P16-B — `self_link_idx_opt = Some(effective_end)` when this is a
    // self-link close. Used to gate the new tail emit arm + override
    // `do_internal_loop` (the trace is designed to loop).
    let self_link_idx_opt: Option<(usize, SelfRecKind)> = match end_idx_opt {
        Some((i, TraceEnd::SelfLink(kind))) => Some((i, kind)),
        _ => None,
    };
    // v2.0 Track-R R3a — `downrec_idx_opt` = `Some(effective_end, return_pc,
    // target_proto_id, depth_delta)` when the down-rec catch tripped. R3a
    // only collects the idx + close marker payload; the tail emit shares
    // the SelfLink path's R1 safe-deopt code (R3b lifts to a real
    // back-edge). Diagnostic only for R3a — the dispatch_off label
    // `"self-link-retf-r1"` stays the same so R2's close-cause counters
    // see the close-cause taxonomy that R3b will branch off.
    let downrec_idx_opt: Option<(usize, u32, usize, u8)> = match end_idx_opt {
        Some((
            i,
            TraceEnd::DownRec {
                return_pc,
                target_proto_id,
                depth_delta,
            },
        )) => Some((i, return_pc, target_proto_id, depth_delta)),
        _ => None,
    };

    // Pre-emit verification. Any op outside the step-5 contract
    // bails so the trace becomes a no-op (the recorder counts it
    // toward the head PC's failure count and won't re-record
    // unless the back-edge counter rolls over again).
    //
    // `consumed_by_cmp[i] = true` marks ops[i] as a `Jmp` whose
    // sole role is to be the recorded post-cmp branch — the cmp's
    // `brif` already carries its control transfer, so emit skips
    // the Jmp's IR entirely.
    let mut consumed_by_cmp = vec![false; effective_end];
    // Parallel to record.ops: for each cmp op, which direction
    // was recorded? `None` for non-cmps; set by the pre-emit pass
    // and consumed by body emit.
    let mut cmp_dirs: Vec<Option<CmpDir>> = vec![None; effective_end];
    checkpoint("pre:cmp-dirs-loop");
    for (i, rop) in record.ops[..effective_end].iter().enumerate() {
        // Folded math-fold ops are validated by the matcher above;
        // the per-op contract here would reject GetTabUp /
        // GetField / Move (dst > A semantic) so skip them.
        if folded_ops[i] {
            continue;
        }
        // P12-S4-step3b — depth>0 ops are allowed inside the inline
        // self-recursion path. `end_idx_opt` already guards the path
        // (cmp@d>0 / ForLoop@d>0 / non-self Call / proto mismatch /
        // depth past MAX_INLINE_DEPTH all close the trace before they
        // hit emit), so any op reaching this point with depth>0 is
        // a same-proto inline body op the lowerer can handle.
        // P13-S13-G v2.6 — capture op_id BEFORE per-op checks for
        // failure-phase narrowing.
        set_last_op_id(rop.inst.op() as u8);
        if !std::ptr::eq(rop.proto.as_ptr(), head_proto.as_ptr()) {
            checkpoint("bail:cmp-dirs-cross-proto-op");
            return None;
        }
        let op = rop.inst.op();
        // P12-S4-step3b — self-recursive Op::Call inside the inline
        // path emits no IR (the next op shifts to the callee window
        // via op_offsets). It's not in `is_whitelisted_step5`, so
        // accept it explicitly when depth>0 OR when the next op is
        // at depth+1 (the recorder's self-recursive marker).
        if matches!(op, Op::Call) {
            // The terminator pass already let this Op::Call past as
            // a self-recursive call. Skip the whitelist check.
            continue;
        }
        // P12-S4-step3b — Op::Return0 / Op::Return1 at depth>0 are
        // the inline path's unwind ops. They're not in the legacy
        // step-5 whitelist (it only handled depth=0 traces with no
        // return semantics); admit them when depth>0.
        if rop.inline_depth > 0 && matches!(op, Op::Return0 | Op::Return1) {
            // Bound the A operand for Return1 — Return0 has no A read.
            if matches!(op, Op::Return1) && (rop.inst.a() as usize) >= max_stack {
                checkpoint("bail:cmp-dirs-Return1-a-oob");
                return None;
            }
            continue;
        }
        if !is_whitelisted_step5(op) {
            checkpoint("bail:cmp-dirs-op-not-whitelisted");
            return None;
        }
        // P12-S11-A — Op::GetField is lowered standalone via
        // luna_jit_table_get_field (string key from Proto.consts).
        // v1.2 D3 Path B — Op::GetTabUp also lowered standalone via
        // luna_jit_op_get_tab_up. Both require K[C] = Str at compile
        // time (the const-pool string key is baked into IR). GetTabUp
        // additionally pins B = upvalue index in the trace head
        // closure; the helper resolves it via JIT_CL TLS at runtime.
        if matches!(op, Op::GetTabUp) {
            let cx = rop.inst.c() as usize;
            if cx >= head_proto.consts.len()
                || !matches!(head_proto.consts[cx], luna_core::runtime::Value::Str(_))
            {
                checkpoint("bail:cmp-dirs-GetTabUp-key-not-str");
                return None;
            }
        }
        // P12-S11-A — Op::SetField uses K[B] as string key,
        // Op::GetField uses K[C]. Pre-emit verifies the const is Str.
        if matches!(op, Op::SetField) {
            let bx = rop.inst.b() as usize;
            if bx >= head_proto.consts.len()
                || !matches!(head_proto.consts[bx], luna_core::runtime::Value::Str(_))
            {
                return None;
            }
        }
        if matches!(op, Op::GetField) {
            let cx = rop.inst.c() as usize;
            if cx >= head_proto.consts.len()
                || !matches!(head_proto.consts[cx], luna_core::runtime::Value::Str(_))
            {
                {
                    checkpoint("bail:cmp-dirs-body-other");
                    return None;
                }
            }
        }
        let ins = rop.inst;
        let a = ins.a() as usize;
        let b = ins.b() as usize;
        let c = ins.c() as usize;
        match op {
            Op::Call => {
                unreachable!("Op::Call only appears at effective_end (truncation guarded above)")
            }
            Op::ForLoop => {
                unreachable!("Op::ForLoop only appears at effective_end (loop-end guarded above)")
            }
            Op::TForLoop => unreachable!(
                "Op::TForLoop only appears at effective_end (close-on-back-edge guarded above)"
            ),
            Op::TForPrep => {
                // P12-S12-B-v2 — generic-for prep: forward `add_pc(bx)`
                // to the body-tail (TForCall). Recorder enters at
                // body_top = head_pc, AFTER TForPrep, so the record
                // body never sees TForPrep in normal pickup; bail if
                // it shows up (mid-body / inline-depth>0 = unsupported
                // shape).
                {
                    checkpoint("bail:cmp-dirs-body-other");
                    return None;
                }
            }
            Op::TForCall => {
                // P12-S12-B-v2 — generic-for body tail. Calls iter
                // via the `luna_jit_op_tforcall` helper. Bounds:
                // helper accesses R[A..A+7] (gen/state/ctrl plus the
                // generator-call window R[A+4..A+6] + space for the
                // first two returns). Restrict to inline_depth = 0
                // (helper reads vm.stack via the trace head's frame
                // base; inline frames aren't pushed during trace IR
                // execution). C field = nvars in [1, 250) per PUC.
                if rop.inline_depth > 0 {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if a + 6 >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                let nvars = ins.c();
                if nvars == 0 || nvars > 250 {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::Concat => {
                // P12-S12-C v1 — N-operand right-associative concat.
                // Helper reads vm.stack[base+A..base+A+B); spill all
                // operand slots in body emit. Restrict to depth=0
                // (helper resolves base via trace head's Lua frame).
                if rop.inline_depth > 0 {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                let n_operands = ins.b() as usize;
                if n_operands < 2 {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                match a.checked_add(n_operands) {
                    Some(end) if end <= max_stack => {}
                    _ => return None,
                }
            }
            Op::GetTabUp => {
                // v1.2 D3 Path B — standalone GetTabUp body bounds.
                // K[C] = Str validated by the upstream cmp-dirs gate;
                // here we just bounds-check the A register.
                if a >= max_stack {
                    checkpoint("bail:cmp-dirs-body-other");
                    return None;
                }
            }
            Op::SetField | Op::GetField => {
                // P12-S11-A — validated above (Str const at K[B] or
                // K[C] respectively); bounds-check the reg operands
                // here.
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if matches!(op, Op::SetField) && c >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if matches!(op, Op::GetField) && b >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::Jmp => {
                // Validated in the second pass below.
            }
            Op::Move => {
                if a >= max_stack || b >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::LoadI => {
                // R[A] := signed-bx immediate. No reg operand
                // beyond A; sBx fits in i32 (decoded from u32 by
                // Inst::sbx) so the i64 conversion is lossless.
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::LoadF => {
                // R[A] := signed-bx immediate as f64.
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::LoadNil => {
                // P12-S6-A2 — R[A..=A+B] := nil. Validate the full
                // range fits in window; emit pass writes iconst(0)
                // per slot.
                match a.checked_add(b) {
                    Some(end) if end < max_stack => {}
                    _ => return None,
                }
            }
            Op::Close => {
                // P12-S7-C — close open upvals at slot ≥ A.
                // S7-C limits to inline_depth=0 (helper reads vm.stack
                // via the trace-head frame's base; inline frames aren't
                // pushed). Bounds check on A.
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if rop.inline_depth > 0 {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::Closure => {
                // P12-S7-A/B — R[A] := closure(proto.protos[Bx]).
                // S7-A handled shared-upval / 0-upval; S7-B adds
                // in_stack upval support via per-upval pre-Closure
                // spill (emit writes vm.stack[base + d.index] from
                // regs[d.index] before calling op_closure helper).
                //
                // Restrictions (S7-B scope):
                // - depth = 0 only: spill writes vm.stack via the
                //   trace-head frame's `base`; inline frames (depth>0)
                //   aren't pushed during trace IR execution, so a
                //   spill at depth>0 would target wrong slots.
                // - Source slot must have a known RegKind (not Unset):
                //   spill needs a tag to pack the i64 payload back to
                //   a Value. Unset would mean trace never wrote the
                //   slot AND entry_tags didn't snapshot it.
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if rop.inline_depth > 0 {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                let bx = ins.bx() as usize;
                if bx >= head_proto.protos.len() {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                let inner = head_proto.protos[bx];
                for d in inner.upvals.iter() {
                    if !d.in_stack {
                        continue;
                    }
                    let src_idx = d.index as usize;
                    if src_idx >= max_stack {
                        {
                            checkpoint("bail:cmp-dirs-body-other");
                            return None;
                        }
                    }
                }
            }
            Op::LoadK => {
                // R[A] := proto.consts[Bx]. Step-8 only lowers
                // Int / Float consts; Str / Bool / Nil need a
                // wider marshalling story.
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                let bx = ins.bx() as usize;
                if bx >= head_proto.consts.len() {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if !matches!(
                    head_proto.consts[bx],
                    luna_core::runtime::Value::Int(_) | luna_core::runtime::Value::Float(_)
                ) {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::Add | Op::Sub | Op::Mul | Op::Div | Op::Pow => {
                if a >= max_stack || b >= max_stack || c >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            // 3-reg Int arith / bitwise ops — same bounds rules as
            // Add/Sub/Mul. Operand-type assumed Int (recorder is
            // trusted); Float / mixed paths would need RegKind
            // tracking like the method JIT.
            Op::IDiv | Op::Mod | Op::BAnd | Op::BOr | Op::BXor | Op::Shl | Op::Shr => {
                if a >= max_stack || b >= max_stack || c >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            // 2-reg unary: `R[A] := op R[B]` — Unm (negation),
            // BNot (bitwise NOT).
            Op::Unm | Op::BNot => {
                if a >= max_stack || b >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            // `if (R[A] == const[B]) ~= K then pc++` — same
            // cmp-then-Jmp shape as Lt/Le/Eq. Const RHS is either
            // an Int (icmp eq) or a Float (fcmp eq).
            Op::EqK => {
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                let bx = ins.b() as usize;
                if bx >= head_proto.consts.len() {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if !matches!(
                    head_proto.consts[bx],
                    luna_core::runtime::Value::Int(_) | luna_core::runtime::Value::Float(_)
                ) {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                // EqK pairs with the same trailing Jmp at
                // cmp_pc + 1 contract as Lt/Le/Eq.
                if i + 1 >= effective_end {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                let next = &record.ops[i + 1];
                if !matches!(next.inst.op(), Op::Jmp) || next.pc != rop.pc + 1 {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                consumed_by_cmp[i + 1] = true;
            }
            Op::Test => {
                // P12-S12-A — `if (not R[A] == k) then pc++`. Same
                // direction inference as cmp ops: next.pc==pc+1 + Jmp
                // → TookJmp (test failed); next.pc==pc+2 → SkippedJmp
                // (test passed, K matched).
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if i + 1 >= effective_end {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                let next = &record.ops[i + 1];
                let took_jmp = matches!(next.inst.op(), Op::Jmp) && next.pc == rop.pc + 1;
                let skipped_jmp = next.pc == rop.pc + 2;
                if took_jmp {
                    consumed_by_cmp[i + 1] = true;
                    cmp_dirs[i] = Some(CmpDir::TookJmp);
                } else if skipped_jmp {
                    let slot = (rop.pc + 1) as usize;
                    let jmp_inst = head_proto.code.get(slot).copied();
                    if !jmp_inst.is_some_and(|x| matches!(x.op(), Op::Jmp)) {
                        {
                            checkpoint("bail:cmp-dirs-body-other");
                            return None;
                        }
                    }
                    cmp_dirs[i] = Some(CmpDir::SkippedJmp);
                } else {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::TestSet => {
                // P12-S12-A-v2 — `if R[B].truthy() == K then R[A]=R[B]
                // else pc++`. R[B] is source; R[A] is move target on
                // test-pass. Direction encoding inverted vs Op::Test:
                //   TookJmp (pc+1 = Jmp) = test passed (no pc++)
                //   SkippedJmp (pc+2)    = test failed (pc++)
                if a >= max_stack || b >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if i + 1 >= effective_end {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                let next = &record.ops[i + 1];
                let took_jmp = matches!(next.inst.op(), Op::Jmp) && next.pc == rop.pc + 1;
                let skipped_jmp = next.pc == rop.pc + 2;
                if took_jmp {
                    consumed_by_cmp[i + 1] = true;
                    cmp_dirs[i] = Some(CmpDir::TookJmp);
                } else if skipped_jmp {
                    let slot = (rop.pc + 1) as usize;
                    let jmp_inst = head_proto.code.get(slot).copied();
                    if !jmp_inst.is_some_and(|x| matches!(x.op(), Op::Jmp)) {
                        {
                            checkpoint("bail:cmp-dirs-body-other");
                            return None;
                        }
                    }
                    cmp_dirs[i] = Some(CmpDir::SkippedJmp);
                } else {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::Lt | Op::Le | Op::Eq => {
                if a >= max_stack || b >= max_stack {
                    {
                        checkpoint("bail:cmp-ab-oob");
                        return None;
                    }
                }
                // Direction inference: peek at the next recorded
                // op's PC relative to the cmp's PC.
                //
                // - next.pc == cmp.pc + 1 (and it's a Jmp): cmp
                //   matched K → fell through to Jmp → executed it.
                //   `TookJmp` direction. consumed_by_cmp marks
                //   the Jmp.
                // - next.pc == cmp.pc + 2: cmp didn't match K →
                //   pc++ skipped the Jmp slot → continued at
                //   pc + 2. `SkippedJmp` direction. The skipped
                //   Jmp isn't in record.ops; we read its target
                //   from `head_proto.code[cmp.pc + 1]` for the
                //   side-exit PC.
                // - Anything else: bail (cross-block path the
                //   step-3 lowerer can't model).
                // P13-S13-G v2.7 — relax to `record.ops.len()`:
                // if the Cmp is at `effective_end - 1`, the
                // terminator at `record.ops[effective_end]`
                // still gives us a pc we can use for direction
                // inference (took_jmp / skipped_jmp). Only bail
                // when there's literally no next recorded op.
                // The took_jmp path's `consumed_by_cmp[i+1]`
                // write is gated by `i + 1 < effective_end` so
                // we don't mark a terminator op as consumed.
                if i + 1 >= record.ops.len() {
                    {
                        checkpoint("bail:cmp-at-record-end");
                        return None;
                    }
                }
                let next = &record.ops[i + 1];
                let took_jmp = matches!(next.inst.op(), Op::Jmp) && next.pc == rop.pc + 1;
                let skipped_jmp = next.pc == rop.pc + 2;
                if took_jmp {
                    if i + 1 < effective_end {
                        consumed_by_cmp[i + 1] = true;
                    }
                    cmp_dirs[i] = Some(CmpDir::TookJmp);
                } else if skipped_jmp {
                    // Verify the slot we'd resume to (the Jmp)
                    // is actually a Jmp in the Proto's bytecode.
                    let slot = (rop.pc + 1) as usize;
                    let jmp_inst = head_proto.code.get(slot).copied();
                    if !jmp_inst.is_some_and(|x| matches!(x.op(), Op::Jmp)) {
                        {
                            checkpoint("bail:cmp-skipped-but-no-jmp-slot");
                            return None;
                        }
                    }
                    cmp_dirs[i] = Some(CmpDir::SkippedJmp);
                } else {
                    {
                        checkpoint("bail:cmp-next-pc-mismatch");
                        return None;
                    }
                }
            }
            // Table ops — A is the dest / table reg per op; B/C may be
            // immediates (SetI's key, GetI's key, NewTable's hints).
            Op::NewTable => {
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::GetI => {
                // R[A] := R[B][C_imm]
                if a >= max_stack || b >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::GetTable => {
                // R[A] := R[B][R[C]]
                if a >= max_stack || b >= max_stack || c >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::SetI => {
                // R[A][B_imm] := R[C]
                if a >= max_stack || c >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::SetTable => {
                // R[A][R[B] or const[B]] := R[C] or const[C].
                // Step-6 only handles the all-reg form (k=false);
                // const RHS goes through different helpers.
                if ins.k() {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if a >= max_stack || b >= max_stack || c >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::SetList => {
                // R[A][C + i] = R[A + i] for i in 1..=B.
                // Step-7 only handles the fixed-count form
                // (B > 0) without the k=true ExtraArg follower
                // (which encodes a >MAX_ABC offset). The element
                // window must fit in the frame.
                if ins.k() || ins.b() == 0 {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if a >= max_stack || a + ins.b() as usize >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::Len => {
                // R[A] := #R[B]
                if a >= max_stack || b >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            Op::GetUpval => {
                // R[A] := UpVal[B]. The upval index B is bounded by
                // head_proto.upvals.len() at compile time.
                if a >= max_stack {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
                if b >= head_proto.upvals.len() {
                    {
                        checkpoint("bail:cmp-dirs-body-other");
                        return None;
                    }
                }
            }
            _ => unreachable!("whitelist gated above"),
        }
    }
    // Jmp validation inside the normal range. A Jmp is OK if it
    // was consumed by a preceding cmp (handled above) or sits at
    // the effective end's last position (the back-edge that closes
    // the loop, or the slot right before an Op::Call truncation —
    // the tail / side-exit emits the control transfer).
    for (i, rop) in record.ops[..effective_end].iter().enumerate() {
        if matches!(rop.inst.op(), Op::Jmp) && !consumed_by_cmp[i] && i + 1 != effective_end {
            return None;
        }
    }

    // Validate the truncating Op::Call (if any). step-5 doesn't
    // verify self-recursion — the recorder is trusted to only
    // feed sound patterns; future steps can tighten this.
    if let Some(call_idx) = call_idx_opt {
        // P12-S4-step3b — call_idx_opt only set for non-self
        // Op::Call at depth 0 (self-recursive inline calls pass
        // through end_idx_opt without truncating; depth>0 closures
        // close via TraceEnd::InlineAbort). The depth check below
        // would now be redundant but is left as a debug assert.
        let rop = &record.ops[call_idx];
        debug_assert_eq!(rop.inline_depth, 0, "TraceEnd::Call only at depth 0");
        if !std::ptr::eq(rop.proto.as_ptr(), head_proto.as_ptr()) {
            return None;
        }
        let a = rop.inst.a() as usize;
        if a >= max_stack {
            return None;
        }
    }

    // Validate the trailing Op::ForLoop (if any). Step-6 only
    // lowers the 5.4+ Int count form — pre-5.3 compares R[A+1]
    // directly against `limit` and uses a different state slot
    // layout, so traces from those dialects bail. The recorder
    // is trusted that R[A..A+3] really do hold Ints at runtime;
    // the dispatcher's all-Int marshal gate enforces that
    // separately on the call boundary.
    // P12-S4-step4b-C-2 — validate Op::Return0/Return1 at depth=0
    // (TraceEnd::Return). Same A bound rule as Call truncation
    // applies to Return1; Return0 has no A read.
    if let Some(return_idx) = return_idx_opt {
        let rop = &record.ops[return_idx];
        debug_assert_eq!(rop.inline_depth, 0, "TraceEnd::Return only at depth 0");
        if !std::ptr::eq(rop.proto.as_ptr(), head_proto.as_ptr()) {
            return None;
        }
        if matches!(rop.inst.op(), Op::Return1) {
            let a = rop.inst.a() as usize;
            if a >= max_stack {
                return None;
            }
        }
    }

    if let Some(for_loop_idx) = for_loop_idx_opt {
        let rop = &record.ops[for_loop_idx];
        debug_assert_eq!(rop.inline_depth, 0, "TraceEnd::ForLoop only at depth 0");
        if !std::ptr::eq(rop.proto.as_ptr(), head_proto.as_ptr()) {
            return None;
        }
        let a = rop.inst.a() as usize;
        match rop.inst.op() {
            Op::ForLoop => {
                if opts.pre53 {
                    return None;
                }
                // ForLoop touches R[A], R[A+1] (count), R[A+2] (step),
                // R[A+3] (visible loop var). All must fit in the frame.
                if a + 3 >= max_stack {
                    return None;
                }
                // Bail on Float ForLoop. Trace JIT's emit at line ~7233
                // reads R[A+1] as Int count + tests `count > 0`. For
                // Float ForLoop (5.4+ Float-counter form), R[A+1] is
                // the LIMIT (Float bits), not a remaining-iteration
                // count. The Int-semantics check would treat the float
                // bits as a large positive integer (always > 0) and
                // loop forever inside the trace. PUC's interp handles
                // Float and Int ForLoop with separate semantics; the
                // trace JIT only emits the Int path correctly.
                // See docs/known-bugs/trace-jit-float-forloop-nested-
                // hang.md for the symptom + investigation.
                if a < record.entry_tags.len()
                    && record.entry_tags[a] == luna_core::runtime::value::raw::FLOAT
                {
                    return None;
                }
            }
            Op::TForLoop => {
                // P12-S12-B-v2 — TForLoop reads R[A+4] (control
                // returned by the iterator) and writes R[A+2] on
                // continue. R[A+4] must fit in the trace's frame.
                if a + 4 >= max_stack {
                    return None;
                }
            }
            _ => unreachable!("for_loop_idx_opt only set for Op::ForLoop / Op::TForLoop"),
        }
    }

    // v1.3 Phase AOT Stage 3 — `module` arrives as `&mut M` from the
    // caller. The JIT wrapper [`try_compile_trace_with_options`]
    // constructs a `JITModule` via [`build_trace_jit_module`]; the AOT
    // pipeline (luna-aot) feeds an `ObjectModule` of its own. The
    // helper-symbol contract is identical (both resolve `luna_jit_*` —
    // the JIT via `JITBuilder::symbol`, the AOT via static link).

    // Helper signatures — declared up front so emit can look them
    // up without re-declaring per call site. Unused declarations
    // get tree-shaken at optimization.
    let mut new_table_sig = module.make_signature();
    new_table_sig.returns.push(AbiParam::new(types::I64));
    let new_table_id = module
        .declare_function("luna_jit_new_table", Linkage::Import, &new_table_sig)
        .ok()?;

    let mut set_int_sig = module.make_signature();
    set_int_sig.params.push(AbiParam::new(types::I64));
    set_int_sig.params.push(AbiParam::new(types::I64));
    set_int_sig.params.push(AbiParam::new(types::I64));
    let set_int_id = module
        .declare_function("luna_jit_table_set_int", Linkage::Import, &set_int_sig)
        .ok()?;

    // P12-S6-A2 — `fn luna_jit_table_set_nil(t: i64, key: i64)`.
    // Returns nothing; writes `Value::Nil` to t[key].
    let mut set_nil_sig = module.make_signature();
    set_nil_sig.params.push(AbiParam::new(types::I64));
    set_nil_sig.params.push(AbiParam::new(types::I64));
    let set_nil_id = module
        .declare_function("luna_jit_table_set_nil", Linkage::Import, &set_nil_sig)
        .ok()?;

    // P12-S7-C — `fn luna_jit_table_set_raw(t, key, raw_bits, tag)`.
    // Writes Value::pack(tag, raw_bits) to t[key]. Used for any
    // src kind other than Int / Nil (Closure / Table / Float / etc.).
    let mut set_raw_sig = module.make_signature();
    set_raw_sig.params.push(AbiParam::new(types::I64));
    set_raw_sig.params.push(AbiParam::new(types::I64));
    set_raw_sig.params.push(AbiParam::new(types::I64));
    set_raw_sig.params.push(AbiParam::new(types::I64));
    let set_raw_id = module
        .declare_function("luna_jit_table_set_raw", Linkage::Import, &set_raw_sig)
        .ok()?;

    // P12-S11-A — `fn luna_jit_table_set_field(t, key_ptr, raw, tag)`.
    let mut set_field_sig = module.make_signature();
    set_field_sig.params.push(AbiParam::new(types::I64));
    set_field_sig.params.push(AbiParam::new(types::I64));
    set_field_sig.params.push(AbiParam::new(types::I64));
    set_field_sig.params.push(AbiParam::new(types::I64));
    let set_field_id = module
        .declare_function("luna_jit_table_set_field", Linkage::Import, &set_field_sig)
        .ok()?;

    // P12-S11-A — `fn luna_jit_table_get_field(t, key_ptr) -> raw`.
    let mut get_field_sig = module.make_signature();
    get_field_sig.params.push(AbiParam::new(types::I64));
    get_field_sig.params.push(AbiParam::new(types::I64));
    get_field_sig.returns.push(AbiParam::new(types::I64));
    let get_field_id = module
        .declare_function("luna_jit_table_get_field", Linkage::Import, &get_field_sig)
        .ok()?;

    // v1.2 D3 Path B — `fn luna_jit_op_get_tab_up(upval_idx, key_ptr) -> raw`.
    let mut get_tab_up_sig = module.make_signature();
    get_tab_up_sig.params.push(AbiParam::new(types::I64));
    get_tab_up_sig.params.push(AbiParam::new(types::I64));
    get_tab_up_sig.returns.push(AbiParam::new(types::I64));
    let get_tab_up_id = module
        .declare_function("luna_jit_op_get_tab_up", Linkage::Import, &get_tab_up_sig)
        .ok()?;

    // P12-S7-A — `fn luna_jit_op_closure(proto_idx: i64) -> i64`.
    // Returns the new Gc<LuaClosure> raw payload bits.
    let mut op_closure_sig = module.make_signature();
    op_closure_sig.params.push(AbiParam::new(types::I64));
    op_closure_sig.returns.push(AbiParam::new(types::I64));
    let op_closure_id = module
        .declare_function("luna_jit_op_closure", Linkage::Import, &op_closure_sig)
        .ok()?;

    // P12-S7-B — `fn luna_jit_spill_to_stack(slot_offset, tag, raw_bits)`.
    // Writes vm.stack[base + slot_offset] = Value::pack(tag, raw).
    let mut spill_sig = module.make_signature();
    spill_sig.params.push(AbiParam::new(types::I64));
    spill_sig.params.push(AbiParam::new(types::I64));
    spill_sig.params.push(AbiParam::new(types::I64));
    let spill_id = module
        .declare_function("luna_jit_spill_to_stack", Linkage::Import, &spill_sig)
        .ok()?;

    // P12-S7-C — `fn luna_jit_op_close(start_offset: i64) -> i64`.
    // Returns 0 (continue) or 1 (deopt — handler would run or
    // pre-existing pending_err).
    let mut op_close_sig = module.make_signature();
    op_close_sig.params.push(AbiParam::new(types::I64));
    op_close_sig.returns.push(AbiParam::new(types::I64));
    let op_close_id = module
        .declare_function("luna_jit_op_close", Linkage::Import, &op_close_sig)
        .ok()?;

    // P12-S12-B-v4 — `fn luna_jit_op_tforcall(abs_offset, nvars,
    // ctrl_out: *mut i64, key_out: *mut i64, val_out: *mut i64) -> i64`.
    // v4 batched: helper fills the three out pointers with raw bits
    // of R[A+2] / R[A+4] / R[A+5] and returns R[A+4]'s tag byte
    // (0..=11) on success, -1 on deopt. Emit reads the buffer via
    // cranelift `stack_load` IR (skips per-slot `stack_load` /
    // `stack_tag` helper calls — the 4-helpers-per-iter overhead
    // v3 was bottlenecked on).
    let mut op_tforcall_sig = module.make_signature();
    op_tforcall_sig.params.push(AbiParam::new(types::I64));
    op_tforcall_sig.params.push(AbiParam::new(types::I64));
    op_tforcall_sig.params.push(AbiParam::new(types::I64));
    op_tforcall_sig.params.push(AbiParam::new(types::I64));
    op_tforcall_sig.params.push(AbiParam::new(types::I64));
    op_tforcall_sig.returns.push(AbiParam::new(types::I64));
    let op_tforcall_id = module
        .declare_function("luna_jit_op_tforcall", Linkage::Import, &op_tforcall_sig)
        .ok()?;

    // P12-S12-B-v2 — `fn luna_jit_stack_load(slot) -> i64` returns
    // raw bits of vm.stack[trace_head_frame.base + slot]. Used to
    // reload trace IR Variables after TForCall mutates vm.stack.
    let mut stack_load_sig = module.make_signature();
    stack_load_sig.params.push(AbiParam::new(types::I64));
    stack_load_sig.returns.push(AbiParam::new(types::I64));
    let stack_load_id = module
        .declare_function("luna_jit_stack_load", Linkage::Import, &stack_load_sig)
        .ok()?;

    // P12-S12-B-v2 — `fn luna_jit_stack_tag(slot) -> i64` returns
    // the raw::* tag byte of vm.stack[trace_head_frame.base + slot].
    // TForLoop tail emit dispatches on this to pick exit-on-Nil /
    // continue-on-Int / deopt-on-other for v2.
    let mut stack_tag_sig = module.make_signature();
    stack_tag_sig.params.push(AbiParam::new(types::I64));
    stack_tag_sig.returns.push(AbiParam::new(types::I64));
    let stack_tag_id = module
        .declare_function("luna_jit_stack_tag", Linkage::Import, &stack_tag_sig)
        .ok()?;

    // P12-S12-C v1 — `fn luna_jit_op_concat(a, n) -> i64`. Returns
    // 0 on success (result at vm.stack[base+a]) or -1 on deopt
    // (metamethod path, type error, length overflow,
    // pre-existing pending_err).
    let mut op_concat_sig = module.make_signature();
    op_concat_sig.params.push(AbiParam::new(types::I64));
    op_concat_sig.params.push(AbiParam::new(types::I64));
    op_concat_sig.returns.push(AbiParam::new(types::I64));
    let op_concat_id = module
        .declare_function("luna_jit_op_concat", Linkage::Import, &op_concat_sig)
        .ok()?;

    // P14-S14-B v2 — `fn luna_jit_str_buf_acquire() -> i64`.
    // Returns a `*mut Vec<u8>` (boxed-leaked); used by buffered
    // accumulator emit at trace fn entry.
    let mut str_buf_acquire_sig = module.make_signature();
    str_buf_acquire_sig.returns.push(AbiParam::new(types::I64));
    let str_buf_acquire_id = module
        .declare_function(
            "luna_jit_str_buf_acquire",
            Linkage::Import,
            &str_buf_acquire_sig,
        )
        .ok()?;

    // P14-S14-B v2 — `fn luna_jit_str_buf_release(buf: i64)`.
    let mut str_buf_release_sig = module.make_signature();
    str_buf_release_sig.params.push(AbiParam::new(types::I64));
    let str_buf_release_id = module
        .declare_function(
            "luna_jit_str_buf_release",
            Linkage::Import,
            &str_buf_release_sig,
        )
        .ok()?;

    // P14-S14-B v2 — `fn luna_jit_str_buf_extend(buf, str_ptr) -> i64`.
    let mut str_buf_extend_sig = module.make_signature();
    str_buf_extend_sig.params.push(AbiParam::new(types::I64));
    str_buf_extend_sig.params.push(AbiParam::new(types::I64));
    str_buf_extend_sig.returns.push(AbiParam::new(types::I64));
    let str_buf_extend_id = module
        .declare_function(
            "luna_jit_str_buf_extend",
            Linkage::Import,
            &str_buf_extend_sig,
        )
        .ok()?;

    // P14-S14-B v2 — `fn luna_jit_str_buf_intern(buf) -> i64`.
    let mut str_buf_intern_sig = module.make_signature();
    str_buf_intern_sig.params.push(AbiParam::new(types::I64));
    str_buf_intern_sig.returns.push(AbiParam::new(types::I64));
    let str_buf_intern_id = module
        .declare_function(
            "luna_jit_str_buf_intern",
            Linkage::Import,
            &str_buf_intern_sig,
        )
        .ok()?;
    // Squelch unused warnings for v2 — v3+ wires call sites.
    let _ = (
        str_buf_acquire_id,
        str_buf_release_id,
        str_buf_extend_id,
        str_buf_intern_id,
    );

    // P12-S12-C v1 — `fn luna_jit_stack_update_raw(slot, raw)`.
    // Used in Op::Concat operand spill for Unset-kind slots.
    let mut update_raw_sig = module.make_signature();
    update_raw_sig.params.push(AbiParam::new(types::I64));
    update_raw_sig.params.push(AbiParam::new(types::I64));
    let update_raw_id = module
        .declare_function(
            "luna_jit_stack_update_raw",
            Linkage::Import,
            &update_raw_sig,
        )
        .ok()?;

    let mut get_int_sig = module.make_signature();
    get_int_sig.params.push(AbiParam::new(types::I64));
    get_int_sig.params.push(AbiParam::new(types::I64));
    get_int_sig.returns.push(AbiParam::new(types::I64));
    let get_int_id = module
        .declare_function("luna_jit_table_get_int", Linkage::Import, &get_int_sig)
        .ok()?;

    let mut len_sig = module.make_signature();
    len_sig.params.push(AbiParam::new(types::I64));
    len_sig.returns.push(AbiParam::new(types::I64));
    let len_id = module
        .declare_function("luna_jit_table_len", Linkage::Import, &len_sig)
        .ok()?;

    // P12-S4-step2b — `fn luna_jit_upval_get(idx: i64) -> i64`. The
    // helper reads `JIT_CL`'s upvals[idx], unpacks to raw payload,
    // returns it as i64. Type tag is lost across the ABI; the
    // dispatcher's exit_tags must use the Untouched fallback
    // (carry the entry tag through) since we can't statically
    // determine what kind of Value an upval holds — historically
    // this used the now-dropped `MoveFrom(u8)` variant (S13-F).
    let mut upval_get_sig = module.make_signature();
    upval_get_sig.params.push(AbiParam::new(types::I64));
    upval_get_sig.returns.push(AbiParam::new(types::I64));
    let upval_get_id = module
        .declare_function("luna_jit_upval_get", Linkage::Import, &upval_get_sig)
        .ok()?;

    // P12-S4-step4b-A — `fn luna_jit_trace_materialize_frames(n: u64,
    // metas: *const FrameMaterializeInfo) -> i64`. Step4b-B fills
    // the real body; step4b-C wires the lowerer's cmp@d>0 emit to
    // call it. Declared up-front so the import is in `module` when
    // step4b-C lands without revisiting the helper-decl block.
    // Unreferenced today; cranelift tree-shakes unused imports at
    // optimization time so this is a zero-cost declaration.
    let mut materialize_sig = module.make_signature();
    materialize_sig.params.push(AbiParam::new(types::I64));
    materialize_sig.params.push(AbiParam::new(types::I64));
    materialize_sig.returns.push(AbiParam::new(types::I64));
    let materialize_id = module
        .declare_function(
            "luna_jit_trace_materialize_frames",
            Linkage::Import,
            &materialize_sig,
        )
        .ok()?;

    // P12-S5-C — `fn luna_jit_materialize_sunk_table(cap: i64,
    // raws_ptr: *const u64, kinds_ptr: *const u8) -> i64`. Emit
    // per cmp side-exit per live Sinkable site: stack-allocates
    // a `cap × 8` raws buffer + a `cap × 1` kinds buffer, fills
    // them from the site's virt slot Variables + virt_kinds tracker,
    // calls this helper, writes the returned `Value::Table` raw
    // bits into the slot's regs Variable so the subsequent
    // `store_back` lands the heap pointer in `reg_state[a]`.
    // P12-S5-C / S11-B-v2 — 7 i64 args:
    //   cap, arr_raws, arr_kinds, n_hash, hash_keys, hash_raws, hash_kinds
    // Returns: heap table raw payload (i64 Gc<Table> ptr).
    let mut mat_sunk_sig = module.make_signature();
    mat_sunk_sig.params.push(AbiParam::new(types::I64));
    mat_sunk_sig.params.push(AbiParam::new(types::I64));
    mat_sunk_sig.params.push(AbiParam::new(types::I64));
    mat_sunk_sig.params.push(AbiParam::new(types::I64));
    mat_sunk_sig.params.push(AbiParam::new(types::I64));
    mat_sunk_sig.params.push(AbiParam::new(types::I64));
    mat_sunk_sig.params.push(AbiParam::new(types::I64));
    mat_sunk_sig.returns.push(AbiParam::new(types::I64));
    let mat_sunk_id = module
        .declare_function(
            "luna_jit_materialize_sunk_table",
            Linkage::Import,
            &mat_sunk_sig,
        )
        .ok()?;

    let mut sig = module.make_signature();
    // Param 0 — reg_state ptr (caller-owned, lives across the call).
    sig.params.push(AbiParam::new(types::I64));
    // Return — continuation PC (head_pc on clean close).
    sig.returns.push(AbiParam::new(types::I64));
    // v1.3 Phase AOT Stage 7 sub-piece 4 — caller-provided name +
    // export linkage when driving the AOT pipeline. The JIT wrapper
    // (`try_compile_trace_with_options`) passes `None`, preserving the
    // original `luna_jit_trace` / `Linkage::Local` shape.
    let (trace_fn_name, trace_fn_linkage) = match aot_fn_name {
        Some(name) => (name, Linkage::Export),
        None => ("luna_jit_trace", Linkage::Local),
    };
    let fn_id = module
        .declare_function(trace_fn_name, trace_fn_linkage, &sig)
        .ok()?;

    let mut ctx = module.make_context();
    ctx.func.signature = sig;
    ctx.func.name = UserFuncName::user(0, fn_id.as_u32());

    let mut fbc = FunctionBuilderContext::new();
    let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut fbc);

    // Two-block layout for the trace body:
    //
    // - `entry` is the function entry — receives the `reg_state`
    //   pointer as block param 0, loads each Lua reg from memory
    //   into a cranelift Variable, then unconditionally jumps to
    //   `body_loop`. The reg-load prelude runs *once* per
    //   dispatcher entry.
    // - `body_loop` is the loop head. The recorded op IR emits
    //   into it (or into successor blocks split off by cmp brifs).
    //   At the trace's clean close — when no `Op::Call` has
    //   truncated it — the tail emits a jump *back* to `body_loop`,
    //   so subsequent iterations stay inside the JIT'd code until
    //   a cmp side-exits. The S3 dispatcher's per-iter marshal
    //   overhead amortizes across however many iterations the
    //   trace runs internally.
    //
    // Cranelift `FunctionBuilder` handles the back-edge phis
    // automatically: every reg's Variable gets a phi at
    // `body_loop`'s entry merging the entry-from-`entry` def
    // (initial load) with the loop-back def (the previous
    // iteration's writes). We delay sealing `body_loop` until
    // after the tail emits its back-edge so cranelift knows both
    // predecessors.
    let entry = bcx.create_block();
    bcx.append_block_params_for_function_params(entry);
    bcx.switch_to_block(entry);
    bcx.seal_block(entry);
    let reg_state = bcx.block_params(entry)[0];

    // P15-A v2-C-A2 — import the `TraceFn` ABI signature once so
    // every side-exit emit can `call_indirect` into a child side
    // trace. Matches the parent's own signature (`(I64) -> I64`).
    let trace_fn_sig_ref: cranelift_codegen::ir::SigRef = {
        let mut sig = module.make_signature();
        sig.params.push(AbiParam::new(types::I64));
        sig.returns.push(AbiParam::new(types::I64));
        bcx.func.import_signature(sig)
    };
    // P15-A v2-C-A2 — singleton GLOBAL side-trace cell shared by
    // every non-INLINE / non-TAG callsite (clean-tail, Call
    // truncation, ForLoop / TForLoop exits, generic deopts). Each
    // such callsite bakes this Box's heap address into its IR.
    // Transported into [`CompiledTrace::global_side_trace_ptr`] at
    // emit end without moving (Box's heap allocation stays put).
    let global_side_trace_box: Box<TCellPtr> = Box::new(TCellPtr::null());
    let _global_side_trace_cell_addr = (&*global_side_trace_box) as *const TCellPtr as i64;

    // P12-S4-step3b — `regs_full` is sized to `window_size_us`, big
    // enough for every inlined frame's register window. Slots
    // [0..max_stack) are loaded from reg_state (caller-marshalled);
    // [max_stack..window_size_us) start as `iconst(0)` so the
    // callee's `GetUpval` / arith fills them. The emit loop below
    // shadows `regs` to the per-op window slice so existing
    // `regs[ins.X()]` indexing automatically shifts across inlined
    // frames without rewriting every access.
    let mut regs_full: Vec<Variable> = Vec::with_capacity(window_size_us);
    for i in 0..window_size_us {
        let v = bcx.declare_var(types::I64);
        if i < max_stack {
            let offset = (i as i32) * 8;
            let v0 = bcx
                .ins()
                .load(types::I64, MemFlags::new(), reg_state, offset);
            bcx.def_var(v, v0);
        } else {
            let z = bcx.ins().iconst(types::I64, 0);
            bcx.def_var(v, z);
        }
        regs_full.push(v);
    }
    // P12-S12-B-v4 — Variable carrying R[A+4]'s tag byte across the
    // TForCall body emit → TForLoop tail emit boundary. TForCall's
    // batched helper returns the tag on success; tail emit reads
    // it via use_var to dispatch on Nil / Int / other instead of
    // calling the `luna_jit_stack_tag` helper. Declared
    // unconditionally — only def_var'd if the trace actually has a
    // TForCall (otherwise unused, cranelift tree-shakes).
    let tforcall_tag_var = bcx.declare_var(types::I64);
    {
        let z = bcx.ins().iconst(types::I64, 0);
        bcx.def_var(tforcall_tag_var, z);
    }

    // v2.0 Track-R R3.3+ sub-1 — depth-relative `base_var` scaffold.
    //
    // RFC: `.dev/rfcs/v2.0-track-r-r3-3-rfc.md` §6 sub-step 1.
    //
    // Sub-1 is SCAFFOLD-ONLY. The Variable is declared at trace head
    // (here, in the entry block immediately after the reg_state load
    // prelude) and initialised to `iconst(0)` as the depth-0 sentinel
    // placeholder. Sub-2 will (a) replace the iconst(0) with a real
    // `reg_state`-relative base address and (b) migrate Op::Move /
    // Op::LoadK / Op::LoadNil arms to load/store via `base_var`
    // instead of `regs_full[off + slot]`. Sub-3 will install the
    // R3d stitch_blk base-shift sequencing (Risk D1.R2 mitigation).
    //
    // Threading: `lower_trace_into_named` is a monolithic function
    // and every op-arm emit lives in its lexical scope, so `base_var`
    // is automatically in scope for sub-2 op-arm migration. No struct
    // refactor needed at this batch — the explicit "compile context"
    // the RFC names is the lexical closure of this fn body, not a
    // separate type.
    //
    // Codegen audit (Risk D1.R1): an unused Variable initialized via
    // a single iconst gets DCE'd by Cranelift's mid-end, so this
    // scaffold has the desired property of being overhead-neutral
    // vs. the pre-sub-1 build. The single `iadd_imm(base_var, 0)` +
    // use below intentionally KEEPS one read live so `cargo asm` can
    // verify the GlobalValue/Variable threading path produces clean
    // codegen (mitigation §8 D1.R1): if Cranelift fails to fold the
    // `+ 0` and emits a spurious add, sub-2 needs the GlobalValue
    // escape route (RFC §8 mitigation) before op-arm migration.
    //
    // Probe: `BASE_VAR_SCAFFOLD_DECLARED` bumps exactly once at the
    // post-def_var point so the regression test
    // `r3_3_sub1_base_var_scaffold.rs` can assert "scaffold ran" on
    // an arbitrary fixture trace without scraping IR text. Bump
    // happens after `def_var` so a `declare_var` panic earlier leaves
    // the counter unchanged.
    let base_var = bcx.declare_var(types::I64);
    {
        let z = bcx.ins().iconst(types::I64, 0);
        bcx.def_var(base_var, z);
        // Mirror the tforcall_tag_var declaration pattern exactly
        // (declare + iconst init + def_var, no anchor use). Cranelift
        // tree-shakes the unused Variable in optimized builds, so the
        // sub-1 scaffold adds zero machine-code residue vs. pre-sub-1.
        // Risk D1.R1 mitigation is deferred to sub-2 where op-arm
        // migration actually exercises `bcx.use_var(base_var)` —
        // that's the codegen surface that matters for the
        // GlobalValue-vs-Variable escape route decision.
        BASE_VAR_SCAFFOLD_DECLARED.with(|c| c.set(c.get().wrapping_add(1)));
    }

    // P12-S5-B — allocate virtual `Variable`s for each Sinkable
    // site that meets v1's sunk-emit criteria. Sites that don't
    // meet the criteria are demoted to Escaped right here so the
    // body emit's site-state check naturally falls through to the
    // existing heap-alloc helper path. Criteria:
    //   - `inline_depth == 0` (trace head's frame only — inline
    //     sinking is S5-C territory, requires extra plumbing for
    //     the materialize helper to address inlined windows)
    //   - `array_cap` in `1..=MAX_SUNK_CAP` (cap = 0 means the
    //     site didn't decode an array part; cap > MAX is a
    //     Cranelift Variable budget guard)
    //   - the site's slot is NOT the trace-terminator `Op::Return1`
    //     R[A] — sinking that case needs the materialize helper
    //     to repack the array into a heap `Gc<Table>` on the way
    //     out, deferred to a follow-up
    //   - the trace's body has NO cmp ops (`Lt`/`Le`/`Eq`/`EqK`) —
    //     a cmp emits a side-exit and the interp resume needs the
    //     heap table; today's sweep escapes all live bindings on
    //     a cmp, but we ALSO need to bail on body cmps that fire
    //     AFTER the site dies (no live binding to escape, but the
    //     trace still has a back-edge candidate). Materialise-on-
    //     deopt + per-side-exit live-set is S5-D territory.
    //
    // Note: looping traces (`opts.internal_loop = true`) that have
    // any cmp in body are already excluded by the sweep escape
    // rule. ForLoop terminators escape via the terminator rule
    // (TraceEnd::ForLoop → all live). So we don't need an explicit
    // `internal_loop` check here.
    const MAX_SUNK_CAP: u32 = 8;
    let return_a_for_sunk_check: Option<u32> = match end_idx_opt {
        Some((idx, TraceEnd::Return)) if idx < record.ops.len() => {
            let term = &record.ops[idx];
            if matches!(term.inst.op(), Op::Return1) && term.inline_depth == 0 {
                Some(term.inst.a())
            } else {
                None
            }
        }
        _ => None,
    };
    // P12-S5-C / S10-B — inline-cmp gate was dropped: inline cmp
    // side-exits (per_exit_inline arm) now call
    // `emit_materialize_live_sunk` to reconstruct live sunk sites
    // before the frame-mat helper pushes inline frames. depth>0
    // cmp no longer demotes sites.
    let mut virt_vars: Vec<Option<Vec<Variable>>> = vec![None; escape.sites.len()];
    let mut virt_kinds: Vec<Option<Vec<RegKind>>> = vec![None; escape.sites.len()];
    let mut sunk_alloc_seen: u32 = 0;
    // P12-S5-C — incremented at each cmp side-exit emit point that
    // materialises ≥1 live Sinkable site. Telemetry only; the
    // dispatcher's runtime materialise calls are not counted here
    // (this is a per-trace static count of emit sites that emit
    // the helper call).
    let mut materialize_emit_count: u32 = 0;
    let mut closure_seen: u32 = 0;
    for (idx, site) in escape.sites.iter_mut().enumerate() {
        if site.state != EscapeState::Sinkable {
            continue;
        }
        // P12-S10-A/B — depth>0 sites are sunk-eligible. Materialise
        // (`emit_materialize_live_sunk`) handles BOTH depth=0 and
        // depth>0 sites at depth=0 cmp arm AND inline cmp
        // (per_exit_inline) arm — `has_inline_cmp` gate dropped in
        // S10-B since inline cmp side-exits now reconstruct live
        // sunk sites. `return_a` check only matters for depth=0
        // (TraceEnd::Return applies at the trace-head frame).
        // P12-S11-B-v1 — total virt slot count = array_cap + hash_keys.
        // - array-only site:    cap = array_cap,           hash = 0
        // - hash-only site:     cap = 0,                   hash = hash_keys.len()
        // - mixed array+hash:   cap = array_cap > 0,       hash > 0
        // - empty (no ops):     cap = 0,                   hash = 0 → demoted below
        let array_cap = site.array_cap as usize;
        let n_hash = site.hash_keys.len();
        let total_slots = array_cap + n_hash;
        if total_slots == 0
            || array_cap > MAX_SUNK_CAP as usize
            || (site.inline_depth == 0 && return_a_for_sunk_check == Some(site.a))
        {
            site.state = EscapeState::Escaped;
            continue;
        }
        // P12-S11-B-v2 — hash slot materialise is now plumbed into
        // emit_materialize_live_sunk (extended helper signature
        // carries hash_keys + hash_raws + hash_kinds buffers); the
        // S11-B-v1 conservative has_any_cmp gate is no longer
        // required. Hash sites survive cmp side-exits via
        // table.set(Value::Str(key), ...) at materialise time.
        let mut vars = Vec::with_capacity(total_slots);
        for _ in 0..total_slots {
            let v = bcx.declare_var(types::I64);
            let z = bcx.ins().iconst(types::I64, 0);
            bcx.def_var(v, z);
            vars.push(v);
        }
        virt_vars[idx] = Some(vars);
        virt_kinds[idx] = Some(vec![RegKind::Unset; total_slots]);
        sunk_alloc_seen += 1;
    }

    // P14-S14-B v4-part2 — if an active_accum is in play,
    // declare buf_var, emit acquire IR, and populate flush_ctx
    // with Some(FlushCtx { ... }). All 19 existing
    // emit_store_back_and_return_* call sites then auto-flush
    // (intern → def_var(accum_slot) → release) before storing
    // back to reg_state.
    if let Some(ref ba) = active_accum {
        let buf_var = bcx.declare_var(types::I64);
        let acquire_ref = module.declare_func_in_func(str_buf_acquire_id, bcx.func);
        let intern_ref = module.declare_func_in_func(str_buf_intern_id, bcx.func);
        let release_ref = module.declare_func_in_func(str_buf_release_id, bcx.func);
        let extend_ref = module.declare_func_in_func(str_buf_extend_id, bcx.func);
        let call_inst = bcx.ins().call(acquire_ref, &[]);
        let ptr = bcx.inst_results(call_inst)[0];
        bcx.def_var(buf_var, ptr);
        // P14-S14-B v4 — prepend the accumulator slot's current
        // bytes into the buffer. The dispatcher always fires on
        // iter 2+ (interp's TForLoop trigger fires AFTER iter 1's
        // body has run), so by the time the trace fn entry
        // executes, `R[accum_slot]` already holds the result of
        // `s` after iter 1 (= entry_s_initial + piece_1). Without
        // this prepend, the flush at exit produces only iter 2..N
        // bytes; the test workload `s = '[' .. iter1 .. ...` loses
        // the leading `[piece_1`. Net effect: buf = accum_slot's
        // entry bytes + iter 2..N piece bytes; flush intern's all
        // bytes; correct result.
        let accum_raw = bcx.use_var(regs_full[ba.accum_slot as usize]);
        let buf_ptr = bcx.use_var(buf_var);
        let _ = bcx.ins().call(extend_ref, &[buf_ptr, accum_raw]);
        flush_ctx = Some(FlushCtx {
            buf_var,
            accum_slot: ba.accum_slot,
            intern_ref,
            release_ref,
        });
    }

    let body_loop = bcx.create_block();
    bcx.ins().jump(body_loop, &[]);
    bcx.switch_to_block(body_loop);
    // Intentionally NOT sealed: the tail's clean-close back-edge
    // adds a second predecessor below.

    // Per-reg current kind. Initialise from the recorder's
    // entry-tag snapshot; writers below refine. Unset slots fall
    // back to Int semantics in the arith / cmp emit — that
    // matches the legacy all-Int behavior for traces built from
    // test harnesses that pass an empty entry_tags vec.
    // P12-S4-step3b — sized to `window_size_us` (mirrors `regs_full`).
    // [0..max_stack) seeded from entry_tags; [max_stack..) start Unset
    // because the dispatcher zero-initialises the corresponding
    // reg_state slots and trace IR fills them via writers.
    let mut current_kinds: Vec<RegKind> = (0..window_size_us)
        .map(|i| {
            if i < max_stack && i < record.entry_tags.len() {
                RegKind::from_entry_tag(record.entry_tags[i]).unwrap_or(RegKind::Unset)
            } else {
                RegKind::Unset
            }
        })
        .collect();
    let mut dispatchable: bool = true;
    // P13-S13-G v2.5 — the first emit-pass site that flips
    // dispatchable to false wins this label; CompiledTrace
    // exposes it via `dispatch_off_reason` for probe diagnostics.
    let mut dispatch_off_reason: Option<&'static str> = None;
    // P12-S4-step2c — per-side-exit RegKind snapshot. Pushed at each
    // true side-exit emit site (Lt/Le/Eq + Jmp) so later writers
    // (e.g. `Op::GetUpval` whose result we infer as `Closure`) don't
    // pollute the side-exit's restore with a tag the slot hasn't
    // actually become at that exit. The clean-tail and call-truncation
    // paths reuse the final `current_kinds` via `ct.exit_tags`.
    // P15-A v2-C-A2 — 3rd element is the per-entry `Box<Cell<*const
    // u8>>` whose heap address is baked into the corresponding
    // emit_store_back_and_return_pc callsite. Allocated at each push
    // site BEFORE the helper call so the IR's `iconst`-baked address
    // exists. Transported through into `tags_side_trace_ptrs` at the
    // end of emit (the cell never moves).
    let mut per_exit_kinds: Vec<(u32, Vec<RegKind>, Box<TCellPtr>)> = Vec::new();
    // P12-S4-step4b-C-2 — per inline cmp@d>0 side-exit. Each entry
    // is built at the cmp emit site and includes the side-exit PC,
    // a window-sized exit-tag snapshot, and the frame-mat chain. The
    // IR encodes `(site_idx + 1)` in the upper 32 bits of the
    // returned i64 so the dispatcher can pick the right entry
    // without colliding on shared cont_pc values (fib's cmp@d=0
    // through cmp@d=4 all side-exit to the same PC).
    // P15-A v2-C-A2 — 5th element is the per-site `Box<Cell<*const
    // u8>>` whose heap address is baked into the IR's
    // `emit_store_back_and_return_site` gate. Allocated at each push
    // site BEFORE the helper call (the v2-B pattern: address is
    // stable across `Vec → Rc<[]>` moves because Box transfers
    // ownership without moving the heap cell).
    let mut per_exit_inline_vec: Vec<(
        u32,
        u32,
        Vec<RegKind>,
        TArc<[FrameMaterializeInfo]>,
        Box<TCellPtr>,
    )> = Vec::new();
    // Live call stack mirror — push on self-recursive `Op::Call`,
    // pop on `Op::Return0/1` at depth>0. Each frame's `base_offset`
    // and `pc` (= caller's Call.pc + 1) are stamped at push time;
    // when snapshotting at a cmp@d>0 site, the innermost frame's
    // `pc` is overwritten with the actual side-exit PC so the helper
    // pushes the right resume point without needing a dispatcher
    // post-hoc fix-up.
    let mut call_chain: Vec<FrameMaterializeInfo> = Vec::new();

    // --- emit body
    //
    // Cranelift's `FunctionBuilder` tracks the "current" block
    // internally; every `bcx.ins()` emits into whichever block was
    // last `switch_to_block`'d. A cmp's `brif` forks the current
    // block to a `continue_blk` and a `side_exit_blk`; after
    // emitting the side-exit and switching back to `continue_blk`,
    // subsequent ops land there. By the end of the loop the
    // "current" block is whatever the last cmp's continue branch
    // pointed at (or the entry block if no cmps fired).
    //
    // Only the *normal* range (`record.ops[..effective_end]`) is
    // emitted. If `Op::Call` truncates the trace, the tail emits
    // a side-exit at the Call's PC instead of the head_pc close.
    // Path C IR-density attack #1 — memoize GetUpval(idx) per dispatch.
    // For self-recursive traces (fib, factorial, etc.), the trace head
    // is entered with one closure and `JIT_CL` stays pinned to it for
    // the entire dispatch; all inlined-depth GetUpval(idx) calls return
    // the same value. Hoist the helper call to the first occurrence and
    // reuse the cached SSA value at later sites (in cranelift-dominated
    // blocks). For fib's 3-deep inline trace, this cuts 4 helper calls
    // to 1 per dispatch (~60 cycles saved × 163k dispatches ≈ 3-5 ms,
    // ~10-15% win).
    //
    // SAFETY of memoization:
    // - Cache invalidation: none required within a single trace
    //   dispatch — `JIT_CL` is pinned at entry and unchanged through
    //   the entire trace body. The first GetUpval(idx) call materializes
    //   the value; subsequent reads of the same idx are exact duplicates.
    // - Cross-block validity: cached values are stored in a Variable
    //   (via def_var / use_var); cranelift's FunctionBuilder inserts
    //   phis as needed for cross-block reads.
    // - Side-exit safety: the first occurrence may be in a block reached
    //   only on the recursive path (e.g. block2 in fib). If a side-exit
    //   fires BEFORE that block (e.g. head-fail base case in block3),
    //   the cache is never populated and reuse never happens — correct.
    let mut upval_cache: std::collections::HashMap<u32, Variable> =
        std::collections::HashMap::new();
    // Path C #2 experiment skipped — iconst memoization adds little
    // since the arm64 backend folds `iconst+isub`/`iconst+icmp` into
    // immediate-form instructions at codegen. See layer-6 doc §3 for
    // attack #2 candidates that ARE structural (block3 dead-slot store
    // elimination, side-exit materialize call ABI consolidation).
    checkpoint("pre:main-emit-loop");
    for (i, rop) in record.ops[..effective_end].iter().enumerate() {
        // P12-S4-step3b — `off` is the start of this op's register
        // window inside reg_state_buf. `regs` is shadowed to the
        // matching slice of `regs_full`, so existing `regs[ins.X()]`
        // indexing auto-shifts across inlined frames. `current_kinds`
        // is NOT shadowed (mut sub-slice would block Return1's
        // cross-window write) — emit code reads/writes via the full
        // Vec with explicit `off + X` indexing.
        let off = op_offsets[i] as usize;
        let regs: &[Variable] = &regs_full[off..off + max_stack];
        // P14-S14-B v4-part2 — body emit handler for the 4-op
        // string-accumulator idiom. Skip the 2 pre-Moves + the
        // post-Move (they're collapsed into the buffered emit).
        // Replace the Concat with `luna_jit_str_buf_extend(buf,
        // piece_raw)` + a deopt branch on -1 (piece wasn't Str
        // → existing __concat metamethod path takes over).
        if let Some(ref ba) = active_accum
            && let Some(ref fctx) = flush_ctx
        {
            if i == ba.pre1_idx || i == ba.pre2_idx || i == ba.post_idx {
                continue;
            }
            if i == ba.concat_idx {
                // Read piece slot raw bits + buf ptr.
                let piece_raw = bcx.use_var(regs[ba.piece_slot as usize]);
                let buf_ptr = bcx.use_var(fctx.buf_var);
                let extend_ref = module.declare_func_in_func(str_buf_extend_id, bcx.func);
                let call_inst = bcx.ins().call(extend_ref, &[buf_ptr, piece_raw]);
                let status = bcx.inst_results(call_inst)[0];
                // Branch on -1 (signed less than 0) → deopt.
                let zero = bcx.ins().iconst(types::I64, 0);
                let is_err = bcx.ins().icmp(IntCC::SignedLessThan, status, zero);
                let continue_blk = bcx.create_block();
                let deopt_blk = bcx.create_block();
                bcx.ins().brif(is_err, deopt_blk, &[], continue_blk, &[]);
                // Deopt path: flush buffer + store back + return pc.
                bcx.switch_to_block(deopt_blk);
                bcx.seal_block(deopt_blk);
                emit_store_back_and_return_pc(
                    &mut bcx,
                    &regs_full[..max_stack],
                    reg_state,
                    rop.pc,
                    flush_ctx.as_ref(),
                    0i64,
                    trace_fn_sig_ref,
                    encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                );
                bcx.switch_to_block(continue_blk);
                bcx.seal_block(continue_blk);
                continue;
            }
        }
        if consumed_by_cmp[i] {
            // The cmp at i-1 already accounted for this Jmp via
            // its `brif`'s continue edge; emitting jump IR here
            // would double-jump.
            continue;
        }
        // Math fold emit. Layout (see `math_folds` doc above):
        //
        //   * `Libm1` — folded indices are `start..=start+3`. The
        //     emit fires at `start` (single libm call); the trailing
        //     3 ops are silent.
        //
        //   * `Min2 / Max2` — folded indices are `start`, `start+1`,
        //     and `call_idx`. The emit fires at `call_idx` (the
        //     `Op::Call`) because that's when args have already
        //     been computed into R[A+1] / R[A+2] by the standard
        //     arg-prep ops between GetField and Call. The
        //     `start` (GetTabUp) and `start+1` (GetField) emit
        //     positions are silent — their semantic result (the
        //     resolved `math.<fn>` callable in R[A]) is known
        //     statically and never consumed by anything except
        //     the Call we're collapsing.
        if folded_ops[i] {
            // Resolve which fold this index belongs to: the start
            // (Libm1 emit site or Min2/Max2 silent GetTabUp), the
            // GetField mid-op (Min2/Max2 silent), or the Call
            // (Min2/Max2 emit site).
            let fold = math_folds.iter().find(|f| {
                f.start_idx == i
                    || (matches!(f.kind, FoldKind::Min2 | FoldKind::Max2)
                        && (f.start_idx + 1 == i || f.call_idx == i))
            });
            if let Some(fold) = fold {
                match fold.kind {
                    FoldKind::Libm1 if fold.start_idx == i => {
                        // Declare libm fn fresh per fold (cranelift
                        // dedups by name in the same module).
                        let mut libm_sig = module.make_signature();
                        libm_sig.params.push(AbiParam::new(types::F64));
                        libm_sig.returns.push(AbiParam::new(types::F64));
                        let libm_id = module
                            .declare_function(fold.fn_name, Linkage::Import, &libm_sig)
                            .ok()?;
                        let libm_ref = module.declare_func_in_func(libm_id, bcx.func);
                        // Libm1 always has a Reg arg_src — coerce
                        // to f64 via the existing Int→f64 / bitcast
                        // ladder based on current_kinds.
                        let arg_src = fold.arg_src.expect("Libm1 has arg_src");
                        let FoldArgSrc::Reg { reg: arg_reg } = arg_src;
                        let arg_kind = k_op(&current_kinds, off as u32 + arg_reg);
                        let arg_f64 = if matches!(arg_kind, RegKind::Float) {
                            use_var_f64(&mut bcx, regs, arg_reg)
                        } else {
                            // Treat Int/Unset/Nil as Int → fcvt. The
                            // recorded trace saw a numeric arg or
                            // it wouldn't have closed.
                            let raw = bcx.use_var(regs[arg_reg as usize]);
                            bcx.ins().fcvt_from_sint(types::F64, raw)
                        };
                        let call = bcx.ins().call(libm_ref, &[arg_f64]);
                        let r = bcx.inst_results(call)[0];
                        def_var_f64(&mut bcx, regs[fold.dst_reg as usize], r);
                        current_kinds[off + fold.dst_reg as usize] = RegKind::Float;
                    }
                    FoldKind::Libm1 => {
                        // Libm1 silent trailer (Move / Call) — folded
                        // away, no IR.
                    }
                    FoldKind::Min2 | FoldKind::Max2 if fold.call_idx == i => {
                        // 2-arg min/max. PUC's `math.min(a, b)` preserves
                        // the operand type — both Int → Int result;
                        // either Float → Float result. So we pick the
                        // IR lowering based on the recorded operand
                        // kinds, mirroring PUC's `vm.less_than` +
                        // pick-loser semantics.
                        //
                        //   Int  / Int  → cranelift `imin_s` / `imax_s`
                        //   Float/ Float → cranelift `fmin` / `fmax`
                        //   mixed         → promote Int to Float, use fmin/fmax
                        //
                        // The IEEE-754 NaN handling of `fmin`/`fmax`
                        // matches PUC's `less_than(NaN, x) = false`
                        // fallthrough for the all-numeric paths the
                        // trace recorder admits.
                        let k1 = k_op(&current_kinds, off as u32 + fold.arg1_reg);
                        let k2 = k_op(&current_kinds, off as u32 + fold.arg2_reg);
                        let result_kind = match (k1, k2) {
                            (RegKind::Float, _) | (_, RegKind::Float) => RegKind::Float,
                            _ => RegKind::Int,
                        };
                        if matches!(result_kind, RegKind::Float) {
                            let coerce_to_f64 = |bcx: &mut FunctionBuilder<'_>,
                                                 regs: &[Variable],
                                                 current_kinds: &[RegKind],
                                                 off: usize,
                                                 reg: u32|
                             -> Value {
                                let kind = k_op(current_kinds, off as u32 + reg);
                                if matches!(kind, RegKind::Float) {
                                    use_var_f64(bcx, regs, reg)
                                } else {
                                    let raw = bcx.use_var(regs[reg as usize]);
                                    bcx.ins().fcvt_from_sint(types::F64, raw)
                                }
                            };
                            let a1 =
                                coerce_to_f64(&mut bcx, regs, &current_kinds, off, fold.arg1_reg);
                            let a2 =
                                coerce_to_f64(&mut bcx, regs, &current_kinds, off, fold.arg2_reg);
                            let r = match fold.kind {
                                FoldKind::Min2 => bcx.ins().fmin(a1, a2),
                                FoldKind::Max2 => bcx.ins().fmax(a1, a2),
                                FoldKind::Libm1 => unreachable!(),
                            };
                            def_var_f64(&mut bcx, regs[fold.dst_reg as usize], r);
                            current_kinds[off + fold.dst_reg as usize] = RegKind::Float;
                        } else {
                            // Int / Int — both operands are i64
                            // payloads holding Int values. Use
                            // signed integer min/max so the result
                            // stays Int-tagged.
                            let a1 = bcx.use_var(regs[fold.arg1_reg as usize]);
                            let a2 = bcx.use_var(regs[fold.arg2_reg as usize]);
                            let r = match fold.kind {
                                FoldKind::Min2 => bcx.ins().smin(a1, a2),
                                FoldKind::Max2 => bcx.ins().smax(a1, a2),
                                FoldKind::Libm1 => unreachable!(),
                            };
                            bcx.def_var(regs[fold.dst_reg as usize], r);
                            current_kinds[off + fold.dst_reg as usize] = RegKind::Int;
                        }
                    }
                    FoldKind::Min2 | FoldKind::Max2 => {
                        // Silent: this index is either `start_idx`
                        // (GetTabUp) or `start_idx + 1` (GetField).
                        // The Call's emit will fire at `call_idx`
                        // and produce the fold's IR.
                    }
                }
            }
            continue;
        }
        let ins = rop.inst;
        let op = ins.op();
        match op {
            Op::Jmp => {
                // Trailing back-edge (validated in the pre-emit
                // pass). The tail's `return iconst(head_pc)`
                // carries the control transfer.
            }
            Op::Move => {
                let src = bcx.use_var(regs[ins.b() as usize]);
                bcx.def_var(regs[ins.a() as usize], src);
                current_kinds[off + ins.a() as usize] = k_op(&current_kinds, off as u32 + ins.b());
            }
            Op::LoadI => {
                let imm = ins.sbx() as i64;
                let v = bcx.ins().iconst(types::I64, imm);
                bcx.def_var(regs[ins.a() as usize], v);
                current_kinds[off + ins.a() as usize] = RegKind::Int;
            }
            Op::LoadF => {
                // R[A] := sBx as f64. Bitcast result to i64
                // bit-pattern so the reg's storage stays uniform.
                let f = ins.sbx() as f64;
                let v = bcx.ins().f64const(f);
                def_var_f64(&mut bcx, regs[ins.a() as usize], v);
                current_kinds[off + ins.a() as usize] = RegKind::Float;
            }
            Op::LoadNil => {
                // P12-S6-A2 — R[A..=A+B] := nil. NIL raw payload bits
                // are 0; emit one iconst(0) and def_var it into each
                // target slot, marking current_kinds = Nil so the
                // exit-tag derivation (kinds_to_exit_tags, S6-A1)
                // produces ExitTag::Nil for slots the trace touched.
                let a_us = ins.a() as usize;
                let b_us = ins.b() as usize;
                let zero = bcx.ins().iconst(types::I64, 0);
                for k in 0..=b_us {
                    bcx.def_var(regs[a_us + k], zero);
                    current_kinds[off + a_us + k] = RegKind::Nil;
                }
            }
            Op::LoadK => {
                let bx = ins.bx() as usize;
                let (v, k) = match head_proto.consts[bx] {
                    luna_core::runtime::Value::Int(n) => {
                        (bcx.ins().iconst(types::I64, n), RegKind::Int)
                    }
                    luna_core::runtime::Value::Float(f) => {
                        let fv = bcx.ins().f64const(f);
                        let bits = bcx.ins().bitcast(types::I64, MemFlags::new(), fv);
                        (bits, RegKind::Float)
                    }
                    _ => unreachable!("pre-emit gates Int / Float consts"),
                };
                bcx.def_var(regs[ins.a() as usize], v);
                current_kinds[off + ins.a() as usize] = k;
            }
            Op::Add | Op::Sub | Op::Mul | Op::Div | Op::Pow => {
                let kb = k_op(&current_kinds, off as u32 + ins.b());
                let kc = k_op(&current_kinds, off as u32 + ins.c());
                // Op::Pow always returns Float in Lua 5.4+ (matches
                // `pow(f64, f64) -> f64`); coerce Int operands to
                // Float via fcvt_from_sint.
                if matches!(op, Op::Pow) {
                    let lhs = match kb {
                        RegKind::Float => use_var_f64(&mut bcx, regs, ins.b()),
                        _ => {
                            let raw = bcx.use_var(regs[ins.b() as usize]);
                            bcx.ins().fcvt_from_sint(types::F64, raw)
                        }
                    };
                    let rhs = match kc {
                        RegKind::Float => use_var_f64(&mut bcx, regs, ins.c()),
                        _ => {
                            let raw = bcx.use_var(regs[ins.c() as usize]);
                            bcx.ins().fcvt_from_sint(types::F64, raw)
                        }
                    };
                    let mut pow_sig = module.make_signature();
                    pow_sig.params.push(AbiParam::new(types::F64));
                    pow_sig.params.push(AbiParam::new(types::F64));
                    pow_sig.returns.push(AbiParam::new(types::F64));
                    let pow_id = module
                        .declare_function("pow", Linkage::Import, &pow_sig)
                        .ok()?;
                    let pow_ref = module.declare_func_in_func(pow_id, bcx.func);
                    let call = bcx.ins().call(pow_ref, &[lhs, rhs]);
                    let r = bcx.inst_results(call)[0];
                    def_var_f64(&mut bcx, regs[ins.a() as usize], r);
                    current_kinds[off + ins.a() as usize] = RegKind::Float;
                    continue;
                }
                // Float path when either operand is known-Float.
                // Both must be Float — mixed Int+Float would
                // semantically coerce to Float in Lua, but the
                // trace's kind tracker bails to avoid the
                // ambiguous emit.
                let float_path = matches!(kb, RegKind::Float) || matches!(kc, RegKind::Float);
                if float_path {
                    if !matches!(kb, RegKind::Float) || !matches!(kc, RegKind::Float) {
                        return None;
                    }
                    let lhs = use_var_f64(&mut bcx, regs, ins.b());
                    let rhs = use_var_f64(&mut bcx, regs, ins.c());
                    let r = match op {
                        Op::Add => bcx.ins().fadd(lhs, rhs),
                        Op::Sub => bcx.ins().fsub(lhs, rhs),
                        Op::Mul => bcx.ins().fmul(lhs, rhs),
                        Op::Div => bcx.ins().fdiv(lhs, rhs),
                        _ => unreachable!(),
                    };
                    def_var_f64(&mut bcx, regs[ins.a() as usize], r);
                    current_kinds[off + ins.a() as usize] = RegKind::Float;
                } else {
                    // Op::Div on Int operands would still coerce
                    // to Float in Lua 5.4+. Bail to be safe.
                    if matches!(op, Op::Div) {
                        return None;
                    }
                    let lhs = bcx.use_var(regs[ins.b() as usize]);
                    let rhs = bcx.use_var(regs[ins.c() as usize]);
                    let r = match op {
                        Op::Add => bcx.ins().iadd(lhs, rhs),
                        Op::Sub => bcx.ins().isub(lhs, rhs),
                        Op::Mul => bcx.ins().imul(lhs, rhs),
                        _ => unreachable!(),
                    };
                    bcx.def_var(regs[ins.a() as usize], r);
                    current_kinds[off + ins.a() as usize] = RegKind::Int;
                }
            }
            // 3-reg Int ops. Cranelift's signed div / mod panic on
            // divide-by-zero — Lua wraps the same way (raises an
            // error). The recorder picked a sample run where these
            // didn't fault, but a runtime zero divisor would crash
            // the trace. Future hardening could trap-then-deopt.
            Op::IDiv | Op::Mod | Op::BAnd | Op::BOr | Op::BXor | Op::Shl | Op::Shr => {
                // Int-only ops. Bail if either operand is Float —
                // Lua's IDiv would coerce to Float (different
                // semantics) and bitwise ops on Floats are
                // type-errors at runtime.
                let kb = k_op(&current_kinds, off as u32 + ins.b());
                let kc = k_op(&current_kinds, off as u32 + ins.c());
                if matches!(kb, RegKind::Float) || matches!(kc, RegKind::Float) {
                    return None;
                }
                let lhs = bcx.use_var(regs[ins.b() as usize]);
                let rhs = bcx.use_var(regs[ins.c() as usize]);
                let r = match op {
                    Op::IDiv => bcx.ins().sdiv(lhs, rhs),
                    Op::Mod => bcx.ins().srem(lhs, rhs),
                    Op::BAnd => bcx.ins().band(lhs, rhs),
                    Op::BOr => bcx.ins().bor(lhs, rhs),
                    Op::BXor => bcx.ins().bxor(lhs, rhs),
                    Op::Shl => bcx.ins().ishl(lhs, rhs),
                    Op::Shr => bcx.ins().ushr(lhs, rhs),
                    _ => unreachable!("whitelist gated above"),
                };
                bcx.def_var(regs[ins.a() as usize], r);
                current_kinds[off + ins.a() as usize] = RegKind::Int;
            }
            Op::Unm | Op::BNot => {
                let kb = k_op(&current_kinds, off as u32 + ins.b());
                if matches!(op, Op::BNot) && matches!(kb, RegKind::Float) {
                    return None;
                }
                if matches!(op, Op::Unm) && matches!(kb, RegKind::Float) {
                    // Float negation.
                    let src = use_var_f64(&mut bcx, regs, ins.b());
                    let r = bcx.ins().fneg(src);
                    def_var_f64(&mut bcx, regs[ins.a() as usize], r);
                    current_kinds[off + ins.a() as usize] = RegKind::Float;
                } else {
                    let src = bcx.use_var(regs[ins.b() as usize]);
                    let r = match op {
                        Op::Unm => bcx.ins().ineg(src),
                        Op::BNot => bcx.ins().bnot(src),
                        _ => unreachable!("whitelist gated above"),
                    };
                    bcx.def_var(regs[ins.a() as usize], r);
                    current_kinds[off + ins.a() as usize] = RegKind::Int;
                }
            }
            Op::EqK => {
                // `R[A] == const[B]` — Int and Float consts both
                // valid (pre-emit gated above). Emit icmp eq for
                // Int + Int, fcmp eq for Float + Float; mismatched
                // pairing on the LHS reg's current kind bails.
                let bx = ins.b() as usize;
                let ka = k_op(&current_kinds, off as u32 + ins.a());
                let cond = match head_proto.consts[bx] {
                    luna_core::runtime::Value::Int(n) => {
                        if matches!(ka, RegKind::Float) {
                            return None;
                        }
                        let lhs = bcx.use_var(regs[ins.a() as usize]);
                        let rhs = bcx.ins().iconst(types::I64, n);
                        let int_cc = if ins.k() {
                            IntCC::Equal
                        } else {
                            IntCC::NotEqual
                        };
                        bcx.ins().icmp(int_cc, lhs, rhs)
                    }
                    luna_core::runtime::Value::Float(f) => {
                        if !matches!(ka, RegKind::Float) {
                            return None;
                        }
                        let lhs = use_var_f64(&mut bcx, regs, ins.a());
                        let rhs = bcx.ins().f64const(f);
                        let float_cc = if ins.k() {
                            FloatCC::Equal
                        } else {
                            FloatCC::NotEqual
                        };
                        bcx.ins().fcmp(float_cc, lhs, rhs)
                    }
                    _ => unreachable!("pre-emit gates Int / Float const only"),
                };

                let continue_blk = bcx.create_block();
                let side_exit_blk = bcx.create_block();
                bcx.ins().brif(cond, continue_blk, &[], side_exit_blk, &[]);

                bcx.switch_to_block(side_exit_blk);
                bcx.seal_block(side_exit_blk);
                let side_exit_pc = rop.pc + 2;
                // P12-S4-step4b-C-2 — at depth>0, the side-exit must
                // materialise the inlined frames before the interp can
                // resume at the cmp's PC. See the matching Lt/Le/Eq
                // arm below for the chain-build details.
                if !call_chain.is_empty() {
                    // Capture head's resume pc BEFORE the innermost
                    // override — `call_chain[0].pc` is the outermost
                    // self-rec Call's `pc + 1` (= trace head's
                    // post-Call resume).
                    let head_resume_pc = call_chain[0].pc;
                    let mut snapshot: Vec<FrameMaterializeInfo> = call_chain.clone();
                    if let Some(last) = snapshot.last_mut() {
                        last.pc = side_exit_pc;
                    }
                    let chain_rc: TArc<[FrameMaterializeInfo]> = snapshot.into();
                    let chain_ptr = TArc::as_ptr(&chain_rc) as *const FrameMaterializeInfo as i64;
                    let chain_len = chain_rc.len() as i64;
                    let site_idx = per_exit_inline_vec.len() as u32;
                    // P12-S10-B — materialise live Sinkable sites
                    // BEFORE the frame_materialize_frames helper
                    // pushes the inline frames. The window-sized
                    // snapshot updates in-place so per_exit_inline's
                    // kinds entry reflects materialised slots.
                    let mut kinds_snapshot: Vec<RegKind> = current_kinds.clone();
                    let mat_count = emit_materialize_live_sunk(
                        &mut bcx,
                        &mut module,
                        mat_sunk_id,
                        &escape,
                        &virt_vars,
                        &virt_kinds,
                        &regs_full,
                        &op_offsets,
                        i,
                        &mut kinds_snapshot,
                        head_proto,
                        opts.aot,
                        &mut defined_aot_data,
                    );
                    materialize_emit_count += mat_count;
                    let inline_side_box_0: Box<TCellPtr> = Box::new(TCellPtr::null());
                    let _inline_side_cell_addr_0 = (&*inline_side_box_0) as *const TCellPtr as i64;
                    let chain_for_helper = chain_rc.clone();
                    per_exit_inline_vec.push((
                        side_exit_pc,
                        head_resume_pc,
                        kinds_snapshot,
                        chain_rc,
                        inline_side_box_0,
                    ));
                    let n_arg = bcx.ins().iconst(types::I64, chain_len);
                    let ptr_arg = emit_chain_ptr_arg(
                        &mut module,
                        &mut bcx,
                        &chain_for_helper,
                        chain_ptr,
                        opts.aot,
                        &mut defined_aot_data,
                    );
                    let mat_ref = module.declare_func_in_func(materialize_id, bcx.func);
                    let _ = bcx.ins().call(mat_ref, &[n_arg, ptr_arg]);
                    emit_store_back_and_return_site(
                        &mut bcx,
                        &regs_full[..window_size_us],
                        reg_state,
                        site_idx,
                        side_exit_pc,
                        flush_ctx.as_ref(),
                        0i64,
                        trace_fn_sig_ref,
                    );
                } else {
                    // P12-S5-C / S10-A — materialise every live
                    // Sinkable site at this depth=0 cmp side-exit.
                    // The snapshot carries `RegKind::Table` for each
                    // materialised caller-window slot so the
                    // dispatcher unpacks the heap pointer correctly
                    // on deopt.
                    let mut snapshot: Vec<RegKind> = current_kinds[..max_stack].to_vec();
                    let mat_count = emit_materialize_live_sunk(
                        &mut bcx,
                        &mut module,
                        mat_sunk_id,
                        &escape,
                        &virt_vars,
                        &virt_kinds,
                        &regs_full,
                        &op_offsets,
                        i,
                        &mut snapshot,
                        head_proto,
                        opts.aot,
                        &mut defined_aot_data,
                    );
                    materialize_emit_count += mat_count;
                    let tag_side_box_0: Box<TCellPtr> = Box::new(TCellPtr::null());
                    let _tag_side_cell_addr_0 = (&*tag_side_box_0) as *const TCellPtr as i64;
                    let tag_side_local_0 = per_exit_kinds.len() as u32;
                    per_exit_kinds.push((side_exit_pc, snapshot, tag_side_box_0));
                    // store_back only writes caller window — depth>0 scratch
                    // slots stay out of the dispatcher's reg_state restore.
                    emit_store_back_and_return_pc(
                        &mut bcx,
                        &regs_full[..max_stack],
                        reg_state,
                        side_exit_pc,
                        flush_ctx.as_ref(),
                        0i64,
                        trace_fn_sig_ref,
                        encode_side_sentinel(SIDE_SENT_KIND_TAG, tag_side_local_0),
                    );
                }

                bcx.switch_to_block(continue_blk);
                bcx.seal_block(continue_blk);
            }
            Op::Test => {
                // P12-S12-A v1 / v3 — `if (not R[A] == K) then pc++`.
                //
                // v1: known kind → compile-time fold (`truthy_known`
                // table). Match recorded → no IR; mismatch → bail.
                // v3: Unset → emit runtime guard via
                // `luna_jit_stack_tag(A)` + `(tag > 1) == truthy`
                // check; runtime mismatch → deopt store_back +
                // return test.pc. The Subsequent Jmp (if TookJmp)
                // is consumed_by_cmp by the pre-emit pass.
                let a_kind = k_op(&current_kinds, off as u32 + ins.a());
                let truthy_known: Option<bool> = match a_kind {
                    RegKind::Int
                    | RegKind::Float
                    | RegKind::Table
                    | RegKind::Closure
                    | RegKind::Str => Some(true),
                    RegKind::Nil => Some(false),
                    RegKind::Unset => None,
                };
                let k_bit = ins.k();
                let recorded_passed = matches!(cmp_dirs[i], Some(CmpDir::SkippedJmp));
                if let Some(truthy) = truthy_known {
                    let test_passed = truthy != k_bit;
                    if test_passed != recorded_passed {
                        // Provably can't reproduce recorded
                        // direction; bail compile.
                        return None;
                    }
                    // Test consumed; no IR. Match guaranteed at
                    // compile time.
                } else {
                    // v3 — runtime tag-based truthy guard.
                    let slot_arg = bcx.ins().iconst(types::I64, ins.a() as i64);
                    let stack_tag_ref = module.declare_func_in_func(stack_tag_id, bcx.func);
                    let tag_call = bcx.ins().call(stack_tag_ref, &[slot_arg]);
                    let tag = bcx.inst_results(tag_call)[0];
                    let one = bcx.ins().iconst(types::I64, 1);
                    let is_truthy = bcx.ins().icmp(IntCC::UnsignedGreaterThan, tag, one);
                    // Op::Test: test_passed_runtime = !is_truthy == k_bit
                    let not_truthy = bcx.ins().bxor_imm(is_truthy, 1);
                    let k_bit_const = bcx.ins().iconst(types::I8, k_bit as i64);
                    let test_passed_runtime = bcx.ins().icmp(IntCC::Equal, not_truthy, k_bit_const);
                    let recorded_const = bcx.ins().iconst(types::I8, recorded_passed as i64);
                    let ok = bcx
                        .ins()
                        .icmp(IntCC::Equal, test_passed_runtime, recorded_const);
                    let cont = bcx.create_block();
                    let deopt = bcx.create_block();
                    bcx.ins().brif(ok, cont, &[], deopt, &[]);
                    bcx.switch_to_block(deopt);
                    bcx.seal_block(deopt);
                    emit_store_back_and_return_pc(
                        &mut bcx,
                        &regs_full[..max_stack],
                        reg_state,
                        rop.pc,
                        flush_ctx.as_ref(),
                        0i64,
                        trace_fn_sig_ref,
                        encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                    );
                    bcx.switch_to_block(cont);
                    bcx.seal_block(cont);
                }
            }
            Op::TestSet => {
                // P12-S12-A-v2 / v3 — `if truthy(R[B]) == K then
                // R[A] = R[B] else pc++`.
                //
                // v2: known kind → compile-time fold + emit Move
                // on `TookJmp` recorded path.
                // v3: Unset → emit runtime guard via stack_tag;
                // when match + TookJmp recorded, emit Move under
                // the `cont` block (so deopt path skips the Move).
                let b_kind = k_op(&current_kinds, off as u32 + ins.b());
                let truthy_known: Option<bool> = match b_kind {
                    RegKind::Int
                    | RegKind::Float
                    | RegKind::Table
                    | RegKind::Closure
                    | RegKind::Str => Some(true),
                    RegKind::Nil => Some(false),
                    RegKind::Unset => None,
                };
                let k_bit = ins.k();
                let recorded_passed = matches!(cmp_dirs[i], Some(CmpDir::TookJmp));
                if let Some(truthy) = truthy_known {
                    let test_passed = truthy == k_bit;
                    if test_passed != recorded_passed {
                        return None;
                    }
                    if test_passed {
                        let v = bcx.use_var(regs[ins.b() as usize]);
                        bcx.def_var(regs[ins.a() as usize], v);
                        current_kinds[off + ins.a() as usize] = b_kind;
                    }
                } else {
                    // v3 — runtime guard. Same shape as Op::Test
                    // but the basis is `is_truthy` (not `!is_truthy`).
                    let slot_arg = bcx.ins().iconst(types::I64, ins.b() as i64);
                    let stack_tag_ref = module.declare_func_in_func(stack_tag_id, bcx.func);
                    let tag_call = bcx.ins().call(stack_tag_ref, &[slot_arg]);
                    let tag = bcx.inst_results(tag_call)[0];
                    let one = bcx.ins().iconst(types::I64, 1);
                    let is_truthy = bcx.ins().icmp(IntCC::UnsignedGreaterThan, tag, one);
                    let k_bit_const = bcx.ins().iconst(types::I8, k_bit as i64);
                    let test_passed_runtime = bcx.ins().icmp(IntCC::Equal, is_truthy, k_bit_const);
                    let recorded_const = bcx.ins().iconst(types::I8, recorded_passed as i64);
                    let ok = bcx
                        .ins()
                        .icmp(IntCC::Equal, test_passed_runtime, recorded_const);
                    let cont = bcx.create_block();
                    let deopt = bcx.create_block();
                    bcx.ins().brif(ok, cont, &[], deopt, &[]);
                    bcx.switch_to_block(deopt);
                    bcx.seal_block(deopt);
                    emit_store_back_and_return_pc(
                        &mut bcx,
                        &regs_full[..max_stack],
                        reg_state,
                        rop.pc,
                        flush_ctx.as_ref(),
                        0i64,
                        trace_fn_sig_ref,
                        encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                    );
                    bcx.switch_to_block(cont);
                    bcx.seal_block(cont);
                    if recorded_passed {
                        let v = bcx.use_var(regs[ins.b() as usize]);
                        bcx.def_var(regs[ins.a() as usize], v);
                        current_kinds[off + ins.a() as usize] = RegKind::Unset;
                    }
                }
            }
            Op::Lt | Op::Le | Op::Eq => {
                // Lua semantics: `if (R[A] op R[B]) ~= K then pc++`.
                // Operand kinds must match — Int → icmp; Float →
                // fcmp; mixed → bail.
                //
                // `cmp_dirs[i]` decides which direction the cmp
                // recorded: TookJmp (cond folds K so true ⇒ took
                // Jmp → continue) or SkippedJmp (cond folds !K so
                // true ⇒ skipped Jmp → continue; side-exit goes
                // to the Jmp's target). The pre-emit pass already
                // validated `i+1 < effective_end`.
                let dir = cmp_dirs[i].expect("cmp dir set in pre-emit");
                let invert = matches!(dir, CmpDir::SkippedJmp);
                let k_effective = if invert { !ins.k() } else { ins.k() };
                let ka = k_op(&current_kinds, off as u32 + ins.a());
                let kb = k_op(&current_kinds, off as u32 + ins.b());
                let float_path = matches!(ka, RegKind::Float) || matches!(kb, RegKind::Float);
                let cond = if float_path {
                    if !matches!(ka, RegKind::Float) || !matches!(kb, RegKind::Float) {
                        return None;
                    }
                    let lhs = use_var_f64(&mut bcx, regs, ins.a());
                    let rhs = use_var_f64(&mut bcx, regs, ins.b());
                    let float_cc = match (op, k_effective) {
                        (Op::Lt, true) => FloatCC::LessThan,
                        (Op::Lt, false) => FloatCC::GreaterThanOrEqual,
                        (Op::Le, true) => FloatCC::LessThanOrEqual,
                        (Op::Le, false) => FloatCC::GreaterThan,
                        (Op::Eq, true) => FloatCC::Equal,
                        (Op::Eq, false) => FloatCC::NotEqual,
                        _ => unreachable!("whitelist gated above"),
                    };
                    bcx.ins().fcmp(float_cc, lhs, rhs)
                } else {
                    let lhs = bcx.use_var(regs[ins.a() as usize]);
                    let rhs = bcx.use_var(regs[ins.b() as usize]);
                    let int_cc = match (op, k_effective) {
                        (Op::Lt, true) => IntCC::SignedLessThan,
                        (Op::Lt, false) => IntCC::SignedGreaterThanOrEqual,
                        (Op::Le, true) => IntCC::SignedLessThanOrEqual,
                        (Op::Le, false) => IntCC::SignedGreaterThan,
                        (Op::Eq, true) => IntCC::Equal,
                        (Op::Eq, false) => IntCC::NotEqual,
                        _ => unreachable!("whitelist gated above"),
                    };
                    bcx.ins().icmp(int_cc, lhs, rhs)
                };

                let continue_blk = bcx.create_block();
                let side_exit_blk = bcx.create_block();
                bcx.ins().brif(cond, continue_blk, &[], side_exit_blk, &[]);

                // Side-exit PC depends on the recorded direction:
                //   TookJmp    → interp's `pc++` lands at cmp_pc + 2.
                //   SkippedJmp → interp would have taken the Jmp;
                //                resume at the Jmp's target.
                let side_exit_pc: u32 = match dir {
                    CmpDir::TookJmp => rop.pc + 2,
                    CmpDir::SkippedJmp => {
                        let jmp_pc = (rop.pc + 1) as usize;
                        let jmp_inst = head_proto.code[jmp_pc];
                        let pc_after_jmp = (rop.pc as i64) + 2;
                        (pc_after_jmp + jmp_inst.sj() as i64) as u32
                    }
                };
                bcx.switch_to_block(side_exit_blk);
                bcx.seal_block(side_exit_blk);
                // P12-S4-step4b-C-2 — at depth>0, snapshot the live
                // `call_chain` (each cmp@d>0 site has its OWN chain;
                // the v1 single-global-array attempt looped fib
                // forever because sibling Calls produced wrong
                // chains under the depth-indexed lookup). The
                // innermost frame's pc is overwritten with this
                // site's side-exit PC so the materialize helper
                // stays PC-agnostic — it just pushes whatever
                // metadata says.
                if !call_chain.is_empty() {
                    let head_resume_pc = call_chain[0].pc;
                    let mut snapshot: Vec<FrameMaterializeInfo> = call_chain.clone();
                    if let Some(last) = snapshot.last_mut() {
                        last.pc = side_exit_pc;
                    }
                    let chain_rc: TArc<[FrameMaterializeInfo]> = snapshot.into();
                    let chain_ptr = TArc::as_ptr(&chain_rc) as *const FrameMaterializeInfo as i64;
                    let chain_len = chain_rc.len() as i64;
                    let site_idx = per_exit_inline_vec.len() as u32;
                    // P12-S10-B — materialise live Sinkable sites
                    // (depth=0 + depth>0) before frame-mat helper
                    // pushes the inline frames.
                    let mut kinds_snapshot: Vec<RegKind> = current_kinds.clone();
                    let mat_count = emit_materialize_live_sunk(
                        &mut bcx,
                        &mut module,
                        mat_sunk_id,
                        &escape,
                        &virt_vars,
                        &virt_kinds,
                        &regs_full,
                        &op_offsets,
                        i,
                        &mut kinds_snapshot,
                        head_proto,
                        opts.aot,
                        &mut defined_aot_data,
                    );
                    materialize_emit_count += mat_count;
                    let inline_side_box_1: Box<TCellPtr> = Box::new(TCellPtr::null());
                    let _inline_side_cell_addr_1 = (&*inline_side_box_1) as *const TCellPtr as i64;
                    let chain_for_helper = chain_rc.clone();
                    per_exit_inline_vec.push((
                        side_exit_pc,
                        head_resume_pc,
                        kinds_snapshot,
                        chain_rc,
                        inline_side_box_1,
                    ));
                    let n_arg = bcx.ins().iconst(types::I64, chain_len);
                    let ptr_arg = emit_chain_ptr_arg(
                        &mut module,
                        &mut bcx,
                        &chain_for_helper,
                        chain_ptr,
                        opts.aot,
                        &mut defined_aot_data,
                    );
                    let mat_ref = module.declare_func_in_func(materialize_id, bcx.func);
                    let _ = bcx.ins().call(mat_ref, &[n_arg, ptr_arg]);
                    emit_store_back_and_return_site(
                        &mut bcx,
                        &regs_full[..window_size_us],
                        reg_state,
                        site_idx,
                        side_exit_pc,
                        flush_ctx.as_ref(),
                        0i64,
                        trace_fn_sig_ref,
                    );
                } else {
                    // P12-S5-C / S10-A — materialise-on-deopt for
                    // depth=0 cmp's live Sinkable sites.
                    let mut snapshot: Vec<RegKind> = current_kinds[..max_stack].to_vec();
                    let mat_count = emit_materialize_live_sunk(
                        &mut bcx,
                        &mut module,
                        mat_sunk_id,
                        &escape,
                        &virt_vars,
                        &virt_kinds,
                        &regs_full,
                        &op_offsets,
                        i,
                        &mut snapshot,
                        head_proto,
                        opts.aot,
                        &mut defined_aot_data,
                    );
                    materialize_emit_count += mat_count;
                    let tag_side_box_1: Box<TCellPtr> = Box::new(TCellPtr::null());
                    let _tag_side_cell_addr_1 = (&*tag_side_box_1) as *const TCellPtr as i64;
                    let tag_side_local_1 = per_exit_kinds.len() as u32;
                    per_exit_kinds.push((side_exit_pc, snapshot, tag_side_box_1));
                    emit_store_back_and_return_pc(
                        &mut bcx,
                        &regs_full[..max_stack],
                        reg_state,
                        side_exit_pc,
                        flush_ctx.as_ref(),
                        0i64,
                        trace_fn_sig_ref,
                        encode_side_sentinel(SIDE_SENT_KIND_TAG, tag_side_local_1),
                    );
                }

                // Continue: subsequent ops emit here.
                bcx.switch_to_block(continue_blk);
                bcx.seal_block(continue_blk);
            }
            Op::NewTable => {
                // P12-S5-B — sunk path: skip the heap alloc helper.
                // The site's virt slot Variables (allocated pre-emit)
                // hold the array elements directly. `current_kinds`
                // for the site's slot stays at its entry value
                // (Unset → maps to ExitTag::Untouched, so the
                // dispatcher carries the entry tag in the restore).
                if let Some(OpAction::NewTableSite { site_idx }) = escape.op_actions[i]
                    && escape.sites[site_idx as usize].state == EscapeState::Sinkable
                    && virt_vars[site_idx as usize].is_some()
                {
                    continue;
                }
                let func_ref = module.declare_func_in_func(new_table_id, bcx.func);
                let call = bcx.ins().call(func_ref, &[]);
                let t = bcx.inst_results(call)[0];
                bcx.def_var(regs[ins.a() as usize], t);
                current_kinds[off + ins.a() as usize] = RegKind::Table;
            }
            Op::GetI => {
                // P12-S5-B — sunk path: a GetI from a Sinkable site
                // at a key in `1..=cap` becomes a `use_var` of the
                // matching virt slot Variable, with kind carried
                // from `virt_kinds`.
                if let Some(OpAction::GetIRead { site_idx, key }) = escape.op_actions[i]
                    && escape.sites[site_idx as usize].state == EscapeState::Sinkable
                    && let Some(vars) = virt_vars[site_idx as usize].as_ref()
                {
                    let slot = (key as usize) - 1;
                    let v = bcx.use_var(vars[slot]);
                    bcx.def_var(regs[ins.a() as usize], v);
                    let k = virt_kinds[site_idx as usize]
                        .as_ref()
                        .expect("Sinkable site has virt_kinds")[slot];
                    current_kinds[off + ins.a() as usize] = k;
                    continue;
                }
                let t = bcx.use_var(regs[ins.b() as usize]);
                let k_imm = bcx.ins().iconst(types::I64, ins.c() as i64);
                let func_ref = module.declare_func_in_func(get_int_id, bcx.func);
                let call = bcx.ins().call(func_ref, &[t, k_imm]);
                let v = bcx.inst_results(call)[0];
                bcx.def_var(regs[ins.a() as usize], v);
                // GetX inference: look at the immediate next op.
                let inferred = if i + 1 < effective_end {
                    infer_getx_exit_lookahead(ins.a(), &record.ops[i + 1..effective_end])
                } else {
                    None
                };
                match inferred {
                    Some(ExitTag::Int) => current_kinds[off + ins.a() as usize] = RegKind::Int,
                    Some(ExitTag::Table) => current_kinds[off + ins.a() as usize] = RegKind::Table,
                    Some(ExitTag::Float) => current_kinds[off + ins.a() as usize] = RegKind::Float,
                    _ => {
                        dispatchable = false;
                        dispatch_off_reason = dispatch_off_reason.or(Some("GetI:inference-fail"));
                    }
                }
            }
            Op::GetTable => {
                let t = bcx.use_var(regs[ins.b() as usize]);
                let key = bcx.use_var(regs[ins.c() as usize]);
                let func_ref = module.declare_func_in_func(get_int_id, bcx.func);
                let call = bcx.ins().call(func_ref, &[t, key]);
                let v = bcx.inst_results(call)[0];
                bcx.def_var(regs[ins.a() as usize], v);
                let inferred = if i + 1 < effective_end {
                    infer_getx_exit_lookahead(ins.a(), &record.ops[i + 1..effective_end])
                } else {
                    None
                };
                match inferred {
                    Some(ExitTag::Int) => current_kinds[off + ins.a() as usize] = RegKind::Int,
                    Some(ExitTag::Table) => current_kinds[off + ins.a() as usize] = RegKind::Table,
                    Some(ExitTag::Float) => current_kinds[off + ins.a() as usize] = RegKind::Float,
                    _ => {
                        dispatchable = false;
                        dispatch_off_reason =
                            dispatch_off_reason.or(Some("GetTable:inference-fail"));
                    }
                }
            }
            Op::SetField => {
                // P12-S11-B-v1 — sunk path: when escape sweep tagged
                // SetFieldSunkWrite, def_var the source register into
                // the matching virt slot (array_cap + hash_slot) +
                // propagate the source RegKind into virt_kinds.
                if let Some(OpAction::SetFieldSunkWrite {
                    site_idx,
                    hash_slot,
                }) = escape.op_actions[i]
                    && escape.sites[site_idx as usize].state == EscapeState::Sinkable
                    && virt_vars[site_idx as usize].is_some()
                {
                    let array_cap = escape.sites[site_idx as usize].array_cap as usize;
                    let slot = array_cap + hash_slot as usize;
                    let src_kind = current_kinds[off + ins.c() as usize];
                    let v = bcx.use_var(regs[ins.c() as usize]);
                    let vars = virt_vars[site_idx as usize]
                        .as_ref()
                        .expect("Sinkable site has virt_vars");
                    bcx.def_var(vars[slot], v);
                    let kinds_vec = virt_kinds[site_idx as usize]
                        .as_mut()
                        .expect("Sinkable site has virt_kinds");
                    kinds_vec[slot] = src_kind;
                    continue;
                }
                // P12-S11-A — helper path: R[A][K[B]:string] := R[C].
                let t = bcx.use_var(regs[ins.a() as usize]);
                let key_v = match head_proto.consts[ins.b() as usize] {
                    luna_core::runtime::Value::Str(s) => s,
                    _ => unreachable!("pre-emit gates Str const at K[B]"),
                };
                let key_arg =
                    emit_str_key_arg(module, &mut bcx, key_v, opts.aot, &mut defined_aot_data);
                let val_kind = k_op(&current_kinds, off as u32 + ins.c());
                let val_tag = match val_kind {
                    RegKind::Int => luna_core::runtime::value::raw::INT,
                    RegKind::Float => luna_core::runtime::value::raw::FLOAT,
                    RegKind::Table => luna_core::runtime::value::raw::TABLE,
                    RegKind::Closure => luna_core::runtime::value::raw::CLOSURE,
                    RegKind::Str => luna_core::runtime::value::raw::STR,
                    RegKind::Nil => luna_core::runtime::value::raw::NIL,
                    RegKind::Unset => luna_core::runtime::value::raw::INT,
                };
                let val_raw = bcx.use_var(regs[ins.c() as usize]);
                let tag_arg = bcx.ins().iconst(types::I64, val_tag as i64);
                let func_ref = module.declare_func_in_func(set_field_id, bcx.func);
                bcx.ins().call(func_ref, &[t, key_arg, val_raw, tag_arg]);
            }
            Op::GetField => {
                // P12-S11-B-v1 — sunk path: use_var the virt slot
                // for hash_slot, def_var R[A], propagate kind.
                if let Some(OpAction::GetFieldSunkRead {
                    site_idx,
                    hash_slot,
                }) = escape.op_actions[i]
                    && escape.sites[site_idx as usize].state == EscapeState::Sinkable
                    && let Some(vars) = virt_vars[site_idx as usize].as_ref()
                {
                    let array_cap = escape.sites[site_idx as usize].array_cap as usize;
                    let slot = array_cap + hash_slot as usize;
                    let v = bcx.use_var(vars[slot]);
                    bcx.def_var(regs[ins.a() as usize], v);
                    let k = virt_kinds[site_idx as usize]
                        .as_ref()
                        .expect("Sinkable site has virt_kinds")[slot];
                    current_kinds[off + ins.a() as usize] = k;
                    continue;
                }
                // P12-S11-A — helper path.
                let t = bcx.use_var(regs[ins.b() as usize]);
                let key_v = match head_proto.consts[ins.c() as usize] {
                    luna_core::runtime::Value::Str(s) => s,
                    _ => unreachable!("pre-emit gates Str const at K[C]"),
                };
                let key_arg =
                    emit_str_key_arg(module, &mut bcx, key_v, opts.aot, &mut defined_aot_data);

                // v2.1 Phase 1I.B — table-field IC scaffold.
                //
                // When `LUNA_JIT_FIELD_IC=1` and this op is the
                // recorder-captured snapshot site, emit an inline
                // cache: 4 guards (mt None, nodes.len() == cached,
                // node[slot].key.raw == cached_key_bits,
                // node[slot].val.tag == cached_val_tag) + 1 load
                // of node[slot].val.raw. Guard miss falls through
                // to the existing helper-call path so no new deopt
                // edge is introduced (scaffold-safe rollout).
                //
                // env-OFF default short-circuits on the cached
                // atomic load inside `field_ic_enabled()`; the IC
                // emission produces zero additional IR when the
                // gate is off.
                let ic_active = luna_core::jit::trace_types::field_ic_enabled()
                    && record
                        .field_ic_snapshot
                        .as_ref()
                        .is_some_and(|s| s.op_idx as usize == i);

                let v = if ic_active {
                    let snap = record
                        .field_ic_snapshot
                        .as_ref()
                        .expect("ic_active implies snapshot present");

                    // --- Guards 1 & 2: metatable + nodes.len() ---
                    let mt = bcx.ins().load(
                        types::I64,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        t,
                        super::TABLE_METATABLE_OFFSET as i32,
                    );
                    let zero = bcx.ins().iconst(types::I64, 0);
                    let mt_ok = bcx.ins().icmp(IntCC::Equal, mt, zero);
                    let nodes_len = bcx.ins().load(
                        types::I64,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        t,
                        super::TABLE_NODES_LEN_OFFSET as i32,
                    );
                    let nodes_len_imm = bcx.ins().iconst(types::I64, snap.nodes_len as i64);
                    let len_ok = bcx.ins().icmp(IntCC::Equal, nodes_len, nodes_len_imm);
                    let guards_12 = bcx.ins().band(mt_ok, len_ok);

                    // 3 blocks: fast (guards 3+4 + load), slow
                    // (helper), merge (def_var dst). slow_blk has 2
                    // predecessors (mt/len fail + key/tag fail); we
                    // seal it only after both edges are emitted.
                    let fast_blk = bcx.create_block();
                    let slow_blk = bcx.create_block();
                    let merge_blk = bcx.create_block();
                    bcx.append_block_param(merge_blk, types::I64);

                    bcx.ins().brif(guards_12, fast_blk, &[], slow_blk, &[]);

                    // --- fast: load nodes_ptr, compute node_addr,
                    //     guards 3 & 4, load val_raw ---
                    bcx.switch_to_block(fast_blk);
                    bcx.seal_block(fast_blk);
                    let nodes_ptr = bcx.ins().load(
                        types::I64,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        t,
                        super::TABLE_NODES_PTR_OFFSET as i32,
                    );
                    let node_offset = (snap.slot_idx as usize * super::SIZEOF_NODE) as i64;
                    let node_addr = bcx.ins().iadd_imm(nodes_ptr, node_offset);

                    let key_raw = bcx.ins().load(
                        types::I64,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        node_addr,
                        super::NODE_KEY_RAW_OFFSET as i32,
                    );
                    let key_imm = bcx.ins().iconst(types::I64, snap.key_ptr_bits as i64);
                    let key_ok = bcx.ins().icmp(IntCC::Equal, key_raw, key_imm);

                    let val_tag_i8 = bcx.ins().load(
                        types::I8,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        node_addr,
                        super::NODE_VAL_TAG_OFFSET as i32,
                    );
                    let val_tag = bcx.ins().uextend(types::I64, val_tag_i8);
                    let tag_imm = bcx.ins().iconst(types::I64, snap.cached_val_tag as i64);
                    let tag_ok = bcx.ins().icmp(IntCC::Equal, val_tag, tag_imm);
                    let guards_34 = bcx.ins().band(key_ok, tag_ok);

                    let load_blk = bcx.create_block();
                    bcx.ins().brif(guards_34, load_blk, &[], slow_blk, &[]);

                    bcx.switch_to_block(load_blk);
                    bcx.seal_block(load_blk);
                    let val_raw = bcx.ins().load(
                        types::I64,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        node_addr,
                        super::NODE_VAL_RAW_OFFSET as i32,
                    );
                    bcx.ins().jump(merge_blk, &[val_raw.into()]);

                    // --- slow: fall back to existing helper ---
                    bcx.switch_to_block(slow_blk);
                    bcx.seal_block(slow_blk);
                    let func_ref = module.declare_func_in_func(get_field_id, bcx.func);
                    let call = bcx.ins().call(func_ref, &[t, key_arg]);
                    let v_slow = bcx.inst_results(call)[0];
                    bcx.ins().jump(merge_blk, &[v_slow.into()]);

                    // --- merge ---
                    bcx.switch_to_block(merge_blk);
                    bcx.seal_block(merge_blk);
                    bcx.block_params(merge_blk)[0]
                } else {
                    let func_ref = module.declare_func_in_func(get_field_id, bcx.func);
                    let call = bcx.ins().call(func_ref, &[t, key_arg]);
                    bcx.inst_results(call)[0]
                };
                bcx.def_var(regs[ins.a() as usize], v);

                let inferred = if i + 1 < effective_end {
                    infer_getx_exit_lookahead(ins.a(), &record.ops[i + 1..effective_end])
                } else {
                    None
                };
                match inferred {
                    Some(ExitTag::Int) => current_kinds[off + ins.a() as usize] = RegKind::Int,
                    Some(ExitTag::Table) => current_kinds[off + ins.a() as usize] = RegKind::Table,
                    Some(ExitTag::Float) => current_kinds[off + ins.a() as usize] = RegKind::Float,
                    _ => {
                        dispatchable = false;
                        dispatch_off_reason =
                            dispatch_off_reason.or(Some("GetField:inference-fail"));
                    }
                }
            }
            Op::GetTabUp => {
                // v1.2 D3 Path B — `R[A] := upvals[B][K[C]:string]`.
                // Helper path mirrors GetField's; the sunk-table
                // optimization does NOT apply (upvalue tables are
                // the global env, not trace-internal alloc). Exit-tag
                // inference identical to GetField — peek next op via
                // `infer_getx_exit`.
                let upval_idx_arg = bcx.ins().iconst(types::I64, ins.b() as i64);
                let key_v = match head_proto.consts[ins.c() as usize] {
                    luna_core::runtime::Value::Str(s) => s,
                    _ => unreachable!("pre-emit gates Str const at K[C]"),
                };
                let key_arg =
                    emit_str_key_arg(module, &mut bcx, key_v, opts.aot, &mut defined_aot_data);
                let func_ref = module.declare_func_in_func(get_tab_up_id, bcx.func);
                let call = bcx.ins().call(func_ref, &[upval_idx_arg, key_arg]);
                let v = bcx.inst_results(call)[0];
                bcx.def_var(regs[ins.a() as usize], v);
                let inferred = if i + 1 < effective_end {
                    infer_getx_exit_lookahead(ins.a(), &record.ops[i + 1..effective_end])
                } else {
                    None
                };
                match inferred {
                    Some(ExitTag::Int) => current_kinds[off + ins.a() as usize] = RegKind::Int,
                    Some(ExitTag::Table) => current_kinds[off + ins.a() as usize] = RegKind::Table,
                    Some(ExitTag::Float) => current_kinds[off + ins.a() as usize] = RegKind::Float,
                    _ => {
                        dispatchable = false;
                        dispatch_off_reason =
                            dispatch_off_reason.or(Some("GetTabUp:inference-fail"));
                    }
                }
            }
            Op::SetI => {
                // P12-S8-B — sunk path: when escape sweep tagged
                // SetISunkWrite, def_var the source register into
                // the matching virt slot Variable + propagate the
                // source RegKind into virt_kinds so the next
                // GetIRead restores the right kind into current_kinds.
                if let Some(OpAction::SetISunkWrite { site_idx, key }) = escape.op_actions[i]
                    && escape.sites[site_idx as usize].state == EscapeState::Sinkable
                    && virt_vars[site_idx as usize].is_some()
                {
                    let slot = (key as usize) - 1;
                    let src_kind = current_kinds[off + ins.c() as usize];
                    let v = bcx.use_var(regs[ins.c() as usize]);
                    let vars = virt_vars[site_idx as usize]
                        .as_ref()
                        .expect("Sinkable site has virt_vars");
                    bcx.def_var(vars[slot], v);
                    let kinds_vec = virt_kinds[site_idx as usize]
                        .as_mut()
                        .expect("Sinkable site has virt_kinds");
                    kinds_vec[slot] = src_kind;
                    continue;
                }
                // R[A][B_imm] := R[C] helper path. Dispatch by R[C]
                // kind via emit_table_set (Nil / Int / Closure / etc.).
                let t = bcx.use_var(regs[ins.a() as usize]);
                let k_imm = bcx.ins().iconst(types::I64, ins.b() as i64);
                let val_kind = k_op(&current_kinds, off as u32 + ins.c());
                emit_table_set(
                    &mut bcx,
                    &mut module,
                    set_int_id,
                    set_nil_id,
                    set_raw_id,
                    t,
                    k_imm,
                    val_kind,
                    regs[ins.c() as usize],
                );
            }
            Op::SetTable => {
                // P12-S8-C — sunk path: escape sweep tagged
                // SetTableSunkWrite when the key reg was const-folded
                // to a 1..=cap literal. Emit shape mirrors SetI sunk
                // (def_var virt slot + propagate kind into virt_kinds).
                if let Some(OpAction::SetTableSunkWrite { site_idx, key }) = escape.op_actions[i]
                    && escape.sites[site_idx as usize].state == EscapeState::Sinkable
                    && virt_vars[site_idx as usize].is_some()
                {
                    let slot = (key as usize) - 1;
                    let src_kind = current_kinds[off + ins.c() as usize];
                    let v = bcx.use_var(regs[ins.c() as usize]);
                    let vars = virt_vars[site_idx as usize]
                        .as_ref()
                        .expect("Sinkable site has virt_vars");
                    bcx.def_var(vars[slot], v);
                    let kinds_vec = virt_kinds[site_idx as usize]
                        .as_mut()
                        .expect("Sinkable site has virt_kinds");
                    kinds_vec[slot] = src_kind;
                    continue;
                }
                // R[A][R[B]] := R[C] helper path. Same kind-dispatch
                // as Op::SetI.
                let t = bcx.use_var(regs[ins.a() as usize]);
                let key = bcx.use_var(regs[ins.b() as usize]);
                let val_kind = k_op(&current_kinds, off as u32 + ins.c());
                emit_table_set(
                    &mut bcx,
                    &mut module,
                    set_int_id,
                    set_nil_id,
                    set_raw_id,
                    t,
                    key,
                    val_kind,
                    regs[ins.c() as usize],
                );
            }
            Op::SetList => {
                // P12-S5-B / P12-S9-C — `R[A][C+i] := R[A+i]` for i in
                // 1..=effective_b. effective_b = bytecode B if B>0,
                // else recorder's var_count snapshot (top - A - 1
                // at the op). For sunk path, effective_b == cap
                // (validated in escape sweep).
                let b_bytecode = ins.b() as usize;
                let effective_b = if b_bytecode == 0 {
                    match record.ops[i].var_count {
                        Some(n) => n as usize,
                        // Unreachable on sunk path (escape sweep
                        // already mark_escaped on None). Helper path
                        // bails compile too — None means no live top.
                        None => return None,
                    }
                } else {
                    b_bytecode
                };
                if let Some(OpAction::SetListWrite { site_idx }) = escape.op_actions[i]
                    && escape.sites[site_idx as usize].state == EscapeState::Sinkable
                    && virt_vars[site_idx as usize].is_some()
                {
                    let a = ins.a() as usize;
                    let mut src_vals: Vec<Value> = Vec::with_capacity(effective_b);
                    let mut src_kinds: Vec<RegKind> = Vec::with_capacity(effective_b);
                    for vi in 1..=effective_b {
                        src_vals.push(bcx.use_var(regs[a + vi]));
                        src_kinds.push(current_kinds[off + a + vi]);
                    }
                    let vars = virt_vars[site_idx as usize]
                        .as_ref()
                        .expect("Sinkable site has virt_vars");
                    for (vi, &v) in src_vals.iter().enumerate() {
                        bcx.def_var(vars[vi], v);
                    }
                    let kinds_vec = virt_kinds[site_idx as usize]
                        .as_mut()
                        .expect("Sinkable site has virt_kinds");
                    for (vi, &k) in src_kinds.iter().enumerate() {
                        kinds_vec[vi] = k;
                    }
                    continue;
                }
                // Helper path: same loop with effective_b iters.
                let a = ins.a() as usize;
                let c_off = ins.c() as i64;
                let t = bcx.use_var(regs[a]);
                for ii in 1..=effective_b {
                    let key = bcx.ins().iconst(types::I64, c_off + ii as i64);
                    let src_kind = k_op(&current_kinds, (off + a + ii) as u32);
                    emit_table_set(
                        &mut bcx,
                        &mut module,
                        set_int_id,
                        set_nil_id,
                        set_raw_id,
                        t,
                        key,
                        src_kind,
                        regs[a + ii],
                    );
                }
            }
            Op::Len => {
                // R[A] := #R[B] — call luna_jit_table_len(t) -> i64.
                let t = bcx.use_var(regs[ins.b() as usize]);
                let func_ref = module.declare_func_in_func(len_id, bcx.func);
                let call = bcx.ins().call(func_ref, &[t]);
                let v = bcx.inst_results(call)[0];
                bcx.def_var(regs[ins.a() as usize], v);
                current_kinds[off + ins.a() as usize] = RegKind::Int;
            }
            Op::Closure => {
                // P12-S7-A/B — R[A] := closure(proto.protos[Bx]).
                // Emit per-in_stack-upval spill (S7-B) followed by a
                // single op_closure helper call (S7-A). Spill writes
                // vm.stack[base + d.index] = Value::pack(tag, raw)
                // so the helper's find_or_create_upval captures a
                // live slot. Restrictions enforced in pre-emit:
                // inline_depth == 0 + every in_stack source reg in
                // bounds. RegKind::Unset src → bail (no known tag).
                let bx = ins.bx() as usize;
                let inner = head_proto.protos[bx];
                let spill_ref = module.declare_func_in_func(spill_id, bcx.func);
                for d in inner.upvals.iter() {
                    if !d.in_stack {
                        continue;
                    }
                    let src_idx = d.index as usize;
                    let src_kind = current_kinds[off + src_idx];
                    let tag_byte = match src_kind {
                        RegKind::Int => luna_core::runtime::value::raw::INT,
                        RegKind::Float => luna_core::runtime::value::raw::FLOAT,
                        RegKind::Table => luna_core::runtime::value::raw::TABLE,
                        RegKind::Closure => luna_core::runtime::value::raw::CLOSURE,
                        RegKind::Str => luna_core::runtime::value::raw::STR,
                        RegKind::Nil => luna_core::runtime::value::raw::NIL,
                        RegKind::Unset => {
                            // Unknown tag for this slot — can't safely
                            // pack to a Value. Bail compile; future
                            // sweep extension could recover from
                            // entry_tags but isn't done here.
                            return None;
                        }
                    };
                    let slot_arg = bcx.ins().iconst(types::I64, d.index as i64);
                    let tag_arg = bcx.ins().iconst(types::I64, tag_byte as i64);
                    let raw_arg = bcx.use_var(regs[src_idx]);
                    bcx.ins().call(spill_ref, &[slot_arg, tag_arg, raw_arg]);
                }
                let bx_arg = bcx.ins().iconst(types::I64, ins.bx() as i64);
                let func_ref = module.declare_func_in_func(op_closure_id, bcx.func);
                let call = bcx.ins().call(func_ref, &[bx_arg]);
                let v = bcx.inst_results(call)[0];
                bcx.def_var(regs[ins.a() as usize], v);
                current_kinds[off + ins.a() as usize] = RegKind::Closure;
                closure_seen += 1;
            }
            Op::Close => {
                // P12-S7-C — close open upvals at slot ≥ A.
                //
                // Sequence:
                //  1. Pre-Close spill every slot in [A..max_stack)
                //     whose current_kinds is known (helper's close_from
                //     reads vm.stack[s] to seal each upval, so the
                //     trace's IR Variable values must reach vm.stack
                //     first).
                //  2. Call `luna_jit_op_close(A)` → 0 (continue) or
                //     1 (deopt: handler would run / pre-pending_err).
                //  3. brif on the i64 status: continue_blk falls
                //     through to subsequent ops; deopt_blk does a
                //     full store_back of all regs and returns close_pc
                //     so the interpreter redoes the Op::Close cleanly.
                //
                // close_from is idempotent (open_upvals are popped on
                // first call), so a deopt that re-fires interp's
                // Op::Close → begin_close → close_from sees no work.
                let a_us = ins.a() as usize;
                let spill_ref = module.declare_func_in_func(spill_id, bcx.func);
                for slot in a_us..max_stack {
                    let k = current_kinds[off + slot];
                    let tag_byte = match k {
                        RegKind::Int => luna_core::runtime::value::raw::INT,
                        RegKind::Float => luna_core::runtime::value::raw::FLOAT,
                        RegKind::Table => luna_core::runtime::value::raw::TABLE,
                        RegKind::Closure => luna_core::runtime::value::raw::CLOSURE,
                        RegKind::Str => luna_core::runtime::value::raw::STR,
                        RegKind::Nil => luna_core::runtime::value::raw::NIL,
                        RegKind::Unset => continue,
                    };
                    let slot_arg = bcx.ins().iconst(types::I64, slot as i64);
                    let tag_arg = bcx.ins().iconst(types::I64, tag_byte as i64);
                    let raw_arg = bcx.use_var(regs[slot]);
                    bcx.ins().call(spill_ref, &[slot_arg, tag_arg, raw_arg]);
                }
                let a_arg = bcx.ins().iconst(types::I64, ins.a() as i64);
                let func_ref = module.declare_func_in_func(op_close_id, bcx.func);
                let call = bcx.ins().call(func_ref, &[a_arg]);
                let status = bcx.inst_results(call)[0];
                let continue_blk = bcx.create_block();
                let deopt_blk = bcx.create_block();
                bcx.ins().brif(status, deopt_blk, &[], continue_blk, &[]);
                bcx.switch_to_block(deopt_blk);
                bcx.seal_block(deopt_blk);
                // Deopt: store back full caller window so interp
                // resumes Op::Close with the trace's mid-stream values
                // in vm.stack. dispatcher's pending_err check will
                // unwind without restoring reg_state; interp re-runs
                // Op::Close → idempotent close_from + handler dispatch.
                emit_store_back_and_return_pc(
                    &mut bcx,
                    &regs_full[..max_stack],
                    reg_state,
                    rop.pc,
                    flush_ctx.as_ref(),
                    0i64,
                    trace_fn_sig_ref,
                    encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                );
                bcx.switch_to_block(continue_blk);
                bcx.seal_block(continue_blk);
            }
            Op::GetUpval => {
                // R[A] := UpVal[B]. The helper reads JIT_CL's
                // upvals[B] and returns the raw 8-byte payload.
                // P12-S4-step2c — use-site inference (`infer_upval_exit`)
                // pins the kind when the immediate use is `Op::Call`
                // on R[A] (the call target must be a closure). Any
                // other shape leaves dispatchable=false. Per-side-exit
                // exit_tags (also step 2c) guard side-exits firing
                // BEFORE this GetUpval: they snapshot the pre-GetUpval
                // current_kinds, so those exits restore as Untouched.
                //
                // Path C #1 — memoize per upval idx via `upval_cache`.
                let idx_b = ins.b();
                let v = if let Some(&cached_var) = upval_cache.get(&idx_b) {
                    bcx.use_var(cached_var)
                } else {
                    let idx_arg = bcx.ins().iconst(types::I64, ins.b() as i64);
                    let func_ref = module.declare_func_in_func(upval_get_id, bcx.func);
                    let call = bcx.ins().call(func_ref, &[idx_arg]);
                    let new_v = bcx.inst_results(call)[0];
                    let cache_var = bcx.declare_var(types::I64);
                    bcx.def_var(cache_var, new_v);
                    upval_cache.insert(idx_b, cache_var);
                    new_v
                };
                bcx.def_var(regs[ins.a() as usize], v);
                // Look forward including the terminator (effective_end
                // is the Op::Call's index when truncation applies).
                let upper = effective_end.min(record.ops.len() - 1) + 1;
                let inferred = if i + 1 < upper {
                    infer_upval_exit(ins.a(), &record.ops[i + 1..upper])
                } else {
                    None
                };
                match inferred {
                    Some(ExitTag::Closure) => {
                        current_kinds[off + ins.a() as usize] = RegKind::Closure;
                    }
                    _ => {
                        current_kinds[off + ins.a() as usize] = RegKind::Unset;
                        dispatchable = false;
                        dispatch_off_reason =
                            dispatch_off_reason.or(Some("GetUpval:not-Closure-use"));
                    }
                }
            }
            // P12-S4-step3b — inline self-recursive Call: emit nothing.
            // The recorder's depth bump (next op at depth+1) drives the
            // op_offsets shift; subsequent emit lands in the callee's
            // register window via the `off` shadow.
            //
            // P12-S4-step4b-C-2 — push the callee frame onto `call_chain`
            // so subsequent cmp@d>0 sites can snapshot the chain. The
            // pushed `pc` is the caller's resume PC (Call.pc + 1); the
            // innermost frame's pc is overwritten with the side-exit PC
            // at snapshot time.
            Op::Call => {
                // P16-B — SelfLink close: the LAST recorded op is the
                // Op::Call whose "next" op (the tripping deepest-depth
                // entry) was never captured. Skip the call_chain push
                // for that trailing Call — the SelfLink tail emit
                // computes its bump_off from this Call's offset + A + 1
                // directly. No FrameMaterializeInfo needed because no
                // side-exit can fire inside the tripping callee (it has
                // no recorded body).
                if self_link_idx_opt.is_some() && i + 1 == effective_end {
                    continue;
                }
                // Next op is at depth+1 (recorder invariant for
                // self-recursive entry); its op_offsets entry is the
                // callee's base_offset.
                debug_assert!(
                    i + 1 < effective_end,
                    "self-rec Call must be followed by callee op in effective_end"
                );
                let callee_base = op_offsets[i + 1];
                call_chain.push(FrameMaterializeInfo {
                    base_offset: callee_base,
                    pc: rop.pc + 1,
                    nresults: 1,
                });
            }
            // P12-S4-step3b — inline Return0: callee returns no values
            // back to the caller. The caller's R[call_a..] slots stay
            // whatever the caller had written (Lua semantics: the
            // return values are nil if the caller's call expected
            // more than the callee delivered; here recorder snapshots
            // a single concrete trip so trust the recorded trace).
            // P12-S4-step4b-C-2 — pop the matching call_chain frame.
            Op::Return0 => {
                debug_assert!(
                    !call_chain.is_empty(),
                    "Return0 at depth>0 has a matching frame"
                );
                call_chain.pop();
            }
            // P12-S4-step3b — inline Return1: copy callee's R[A]
            // into the caller's R[call_a]. `op_offsets` for the
            // following ops will revert to the caller's window, but
            // the value lives in `regs_full[caller_off + call_a]`
            // ready for the caller's continuation to read it.
            Op::Return1 => {
                let a_callee = ins.a() as usize;
                let call_a = enclosing_call_a[i]
                    .expect("Return1 at depth>0 has an enclosing Op::Call")
                    as usize;
                // Caller window's offset is below ours by call_a+1
                // (callee R[0] sits at caller R[call_a+1]).
                let caller_off = off
                    .checked_sub(call_a + 1)
                    .expect("op_offsets invariant: callee window > caller window");
                let src_var = regs_full[off + a_callee];
                let dst_var = regs_full[caller_off + call_a];
                let v = bcx.use_var(src_var);
                bcx.def_var(dst_var, v);
                // Propagate the kind so the caller's continuation
                // sees the right type.
                current_kinds[caller_off + call_a] = current_kinds[off + a_callee];
                // P12-S4-step4b-C-2 — pop matching call_chain frame.
                debug_assert!(
                    !call_chain.is_empty(),
                    "Return1 at depth>0 has a matching frame"
                );
                call_chain.pop();
            }
            // P12-S12-B-v2 — generic-for body tail. Sequence:
            //   1. Spill regs[A..=A+2] (iter / state / control) to
            //      vm.stack so the helper's
            //      `vm.stack[A+4..=A+6] = vm.stack[A..=A+2]` copy
            //      sees current trace values (control changes each
            //      iter via TForLoop's R[A+2] = R[A+4] writeback).
            //   2. Call `luna_jit_op_tforcall(A, nvars)`. Status
            //      `< 0` → deopt (Lua-closure iter or runtime err).
            //   3. Continue branch: reload regs[A+2] + regs[A+4..]
            //      from vm.stack so subsequent body iters (after the
            //      back-edge from TForLoop) see iter results.
            //      current_kinds for reloaded slots = Unset; the
            //      first body iter still uses entry-tag kinds, and
            //      TForLoop tail's tag-check guards the back-edge
            //      so runtime types match emit-time assumptions.
            Op::TForCall => {
                let a_us = ins.a() as usize;
                let nvars = ins.c() as i64;
                // P12-S12-B-v5 — ipairs detection. Recorder's TForLoop
                // trigger snapshots `R[A]` if Native; we compare against
                // `ipairs_iter`'s address to specialise emit into inline
                // Table aget IR (skip the `op_tforcall` C call entirely
                // on the hot path).
                let ipairs_addr = luna_core::vm::builtins::ipairs_iter
                    as luna_core::runtime::value::NativeFn
                    as usize;
                let is_ipairs_trace = record.tfor_iter_ptr == Some(ipairs_addr);

                // P12-S12-B-v5 — spill discipline:
                // - non-ipairs case: spill R[A..=A+2] upfront (helper
                //   path runs unconditionally; needs vm.stack populated).
                // - ipairs case: SKIP the upfront spill on the hot
                //   path (R[A] and R[A+1] never change inside the
                //   trace — vm.stack still holds entry values, which
                //   is what the slow_blk helper reads). R[A+2] is
                //   spilled INSIDE slow_blk only, so fast iters pay
                //   nothing.
                let spill_ref = module.declare_func_in_func(spill_id, bcx.func);
                let spill_slot = |bcx: &mut FunctionBuilder<'_>, slot: usize| {
                    let k = current_kinds[off + slot];
                    let tag_byte = match k {
                        RegKind::Int => luna_core::runtime::value::raw::INT,
                        RegKind::Float => luna_core::runtime::value::raw::FLOAT,
                        RegKind::Table => luna_core::runtime::value::raw::TABLE,
                        RegKind::Closure => luna_core::runtime::value::raw::CLOSURE,
                        RegKind::Str => luna_core::runtime::value::raw::STR,
                        RegKind::Nil => luna_core::runtime::value::raw::NIL,
                        RegKind::Unset => return,
                    };
                    let slot_arg = bcx.ins().iconst(types::I64, slot as i64);
                    let tag_arg = bcx.ins().iconst(types::I64, tag_byte as i64);
                    let raw_arg = bcx.use_var(regs[slot]);
                    bcx.ins().call(spill_ref, &[slot_arg, tag_arg, raw_arg]);
                };
                if !is_ipairs_trace {
                    for slot in a_us..=(a_us + 2) {
                        spill_slot(&mut bcx, slot);
                    }
                }

                // The helper-call path (used by slow_blk in the
                // ipairs case + the non-ipairs case wholesale).
                // Allocates the 3-slot buffer, calls the helper,
                // brif-checks the result, def_vars regs + tag from
                // the buffer.
                let emit_helper_call = |bcx: &mut FunctionBuilder<'_>, module: &mut M| -> () {
                    let out_ss =
                        bcx.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
                            cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
                            24,
                            3,
                        ));
                    let ctrl_addr = bcx.ins().stack_addr(types::I64, out_ss, 0);
                    let key_addr = bcx.ins().stack_addr(types::I64, out_ss, 8);
                    let val_addr = bcx.ins().stack_addr(types::I64, out_ss, 16);
                    let a_arg = bcx.ins().iconst(types::I64, a_us as i64);
                    let nvars_arg = bcx.ins().iconst(types::I64, nvars);
                    let func_ref = module.declare_func_in_func(op_tforcall_id, bcx.func);
                    let call_inst = bcx
                        .ins()
                        .call(func_ref, &[a_arg, nvars_arg, ctrl_addr, key_addr, val_addr]);
                    let status_or_tag = bcx.inst_results(call_inst)[0];
                    let zero = bcx.ins().iconst(types::I64, 0);
                    let is_err = bcx.ins().icmp(IntCC::SignedLessThan, status_or_tag, zero);
                    let cont_blk = bcx.create_block();
                    let deopt_blk = bcx.create_block();
                    bcx.ins().brif(is_err, deopt_blk, &[], cont_blk, &[]);
                    bcx.switch_to_block(deopt_blk);
                    bcx.seal_block(deopt_blk);
                    emit_store_back_and_return_pc(
                        bcx,
                        &regs_full[..max_stack],
                        reg_state,
                        rop.pc,
                        flush_ctx.as_ref(),
                        0i64,
                        trace_fn_sig_ref,
                        encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                    );
                    bcx.switch_to_block(cont_blk);
                    bcx.seal_block(cont_blk);
                    bcx.def_var(tforcall_tag_var, status_or_tag);
                    let ctrl_raw = bcx.ins().stack_load(types::I64, out_ss, 0);
                    let key_raw = bcx.ins().stack_load(types::I64, out_ss, 8);
                    let val_raw = bcx.ins().stack_load(types::I64, out_ss, 16);
                    bcx.def_var(regs[a_us + 2], ctrl_raw);
                    bcx.def_var(regs[a_us + 4], key_raw);
                    if (nvars as usize) >= 2 && a_us + 5 < max_stack {
                        bcx.def_var(regs[a_us + 5], val_raw);
                    }
                };

                if is_ipairs_trace {
                    // Inline aget fast path. The recorder confirmed
                    // R[A] = ipairs_iter at trace start. The standard
                    // ipairs loop has R[A+1] = Table (state) and
                    // R[A+2] = Int (control = last seen index).
                    // Per iter: next_i = ctrl + 1; val = t[next_i].
                    // If val is Nil → loop ends; else key = next_i,
                    // val_raw = val's payload.
                    let ctrl = bcx.use_var(regs[a_us + 2]);
                    let t_raw = bcx.use_var(regs[a_us + 1]);
                    let one = bcx.ins().iconst(types::I64, 1);
                    let next_i = bcx.ins().iadd(ctrl, one);
                    let key_m1 = ctrl;

                    let asize = bcx.ins().load(
                        types::I64,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        t_raw,
                        super::TABLE_ASIZE_OFFSET as i32,
                    );
                    let in_range = bcx.ins().icmp(IntCC::UnsignedLessThan, key_m1, asize);
                    let metatable = bcx.ins().load(
                        types::I64,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        t_raw,
                        super::TABLE_METATABLE_OFFSET as i32,
                    );
                    let zero = bcx.ins().iconst(types::I64, 0);
                    let no_meta = bcx.ins().icmp(IntCC::Equal, metatable, zero);
                    let fast_ok = bcx.ins().band(in_range, no_meta);

                    let fast_blk = bcx.create_block();
                    let slow_blk = bcx.create_block();
                    let merge_blk = bcx.create_block();
                    bcx.ins().brif(fast_ok, fast_blk, &[], slow_blk, &[]);

                    // ----- fast_blk: inline aget + populate -----
                    bcx.switch_to_block(fast_blk);
                    bcx.seal_block(fast_blk);
                    let avals_ptr = bcx.ins().load(
                        types::I64,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        t_raw,
                        super::TABLE_ARRAY_PTR_OFFSET as i32,
                    );
                    let three = bcx.ins().iconst(types::I64, 3);
                    let val_off = bcx.ins().ishl(key_m1, three);
                    let val_addr_fast = bcx.ins().iadd(avals_ptr, val_off);
                    let val_raw_fast = bcx.ins().load(
                        types::I64,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        val_addr_fast,
                        0,
                    );
                    let avals_bytes = bcx.ins().ishl(asize, three);
                    let tag_base = bcx.ins().iadd(avals_ptr, avals_bytes);
                    let tag_addr = bcx.ins().iadd(tag_base, key_m1);
                    let val_tag_i8 = bcx.ins().load(
                        types::I8,
                        cranelift_codegen::ir::MemFlags::trusted(),
                        tag_addr,
                        0,
                    );
                    let val_tag = bcx.ins().uextend(types::I64, val_tag_i8);
                    let nil_const = bcx
                        .ins()
                        .iconst(types::I64, luna_core::runtime::value::raw::NIL as i64);
                    let int_const = bcx
                        .ins()
                        .iconst(types::I64, luna_core::runtime::value::raw::INT as i64);
                    let is_nil = bcx.ins().icmp(IntCC::Equal, val_tag, nil_const);
                    // P12-S12-C v3 — runtime val_tag guard. Snapshot
                    // at recorder fire (R[A+5]'s tag) is the
                    // *expected* iter val tag. The trace's
                    // downstream emit (Move propagation, Concat
                    // spill via RegKind::Str etc.) is specialised
                    // to this tag. If a subsequent iter delivers a
                    // different non-Nil tag (mixed-tag array), the
                    // spill would pack stale bits as the snapshot
                    // tag → garbage Value. Guard: `val_tag == Nil
                    // OR val_tag == expected_tag` → continue, else
                    // deopt. Skip the guard when no snapshot is
                    // available (snapshot=None) or when the
                    // snapshot is Nil itself (degenerate).
                    if let Some(expected_tag) = record.tfor_val_tag
                        && expected_tag != luna_core::runtime::value::raw::NIL
                    {
                        let exp_const = bcx.ins().iconst(types::I64, expected_tag as i64);
                        let is_exp = bcx.ins().icmp(IntCC::Equal, val_tag, exp_const);
                        let ok = bcx.ins().bor(is_nil, is_exp);
                        let guard_continue = bcx.create_block();
                        let guard_deopt = bcx.create_block();
                        bcx.ins().brif(ok, guard_continue, &[], guard_deopt, &[]);
                        bcx.switch_to_block(guard_deopt);
                        bcx.seal_block(guard_deopt);
                        emit_store_back_and_return_pc(
                            &mut bcx,
                            &regs_full[..max_stack],
                            reg_state,
                            rop.pc,
                            flush_ctx.as_ref(),
                            0i64,
                            trace_fn_sig_ref,
                            encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                        );
                        bcx.switch_to_block(guard_continue);
                        bcx.seal_block(guard_continue);
                    }
                    let zero_raw = bcx.ins().iconst(types::I64, 0);
                    // R[A+4] = is_nil ? Nil(raw=0) : Int(raw=next_i)
                    let r4_raw = bcx.ins().select(is_nil, zero_raw, next_i);
                    let r4_tag = bcx.ins().select(is_nil, nil_const, int_const);
                    bcx.def_var(regs[a_us + 2], next_i);
                    bcx.def_var(regs[a_us + 4], r4_raw);
                    if a_us + 5 < max_stack {
                        // On the Nil branch, exit_tag[A+5] stays
                        // `Untouched` (no per-side-exit override
                        // for A+5), so the dispatcher restores
                        // using entry_tag. If entry was a Str
                        // slot, packing with raw=0 produces a null
                        // Gc<LuaStr> → panic on the next interp
                        // touch. Preserve the previous regs[A+5]
                        // (= the last non-Nil iter's value) on the
                        // Nil branch so the trace exit restore
                        // sees a real GC pointer.
                        let prev_v5 = bcx.use_var(regs[a_us + 5]);
                        let chosen_v5 = bcx.ins().select(is_nil, prev_v5, val_raw_fast);
                        bcx.def_var(regs[a_us + 5], chosen_v5);
                    }
                    bcx.def_var(tforcall_tag_var, r4_tag);
                    bcx.ins().jump(merge_blk, &[]);

                    // ----- slow_blk: helper fallback -----
                    bcx.switch_to_block(slow_blk);
                    bcx.seal_block(slow_blk);
                    // Spill R[A+2] (ctrl, the only slot that
                    // changes per iter via TForLoop's writeback)
                    // so the helper sees the trace's current
                    // value. R[A]/R[A+1] still hold their entry
                    // values in vm.stack.
                    spill_slot(&mut bcx, a_us + 2);
                    emit_helper_call(&mut bcx, module);
                    bcx.ins().jump(merge_blk, &[]);

                    // ----- merge_blk -----
                    bcx.switch_to_block(merge_blk);
                    bcx.seal_block(merge_blk);
                } else {
                    emit_helper_call(&mut bcx, module);
                }

                current_kinds[off + a_us + 2] = RegKind::Unset;
                current_kinds[off + a_us + 4] = RegKind::Unset;
                if (nvars as usize) >= 2 && a_us + 5 < max_stack {
                    current_kinds[off + a_us + 5] = RegKind::Unset;
                }
            }
            // P12-S12-C v1 — N-operand concat via helper.
            Op::Concat => {
                let a_us = ins.a() as usize;
                let n_operands = ins.b() as usize;
                // Spill every operand slot to vm.stack so the
                // helper's concat_run can read them. For Unset
                // kinds (e.g. Str — RegKind doesn't carry Str)
                // call stack_update_raw which preserves the
                // existing tag and only refreshes the raw bits.
                let spill_ref = module.declare_func_in_func(spill_id, bcx.func);
                let update_raw_ref = module.declare_func_in_func(update_raw_id, bcx.func);
                for slot in a_us..(a_us + n_operands) {
                    let k = current_kinds[off + slot];
                    let slot_arg = bcx.ins().iconst(types::I64, slot as i64);
                    let raw_arg = bcx.use_var(regs[slot]);
                    let tag_byte_opt = match k {
                        RegKind::Int => Some(luna_core::runtime::value::raw::INT),
                        RegKind::Float => Some(luna_core::runtime::value::raw::FLOAT),
                        RegKind::Table => Some(luna_core::runtime::value::raw::TABLE),
                        RegKind::Closure => Some(luna_core::runtime::value::raw::CLOSURE),
                        RegKind::Str => Some(luna_core::runtime::value::raw::STR),
                        RegKind::Nil => Some(luna_core::runtime::value::raw::NIL),
                        RegKind::Unset => None,
                    };
                    if let Some(tag_byte) = tag_byte_opt {
                        let tag_arg = bcx.ins().iconst(types::I64, tag_byte as i64);
                        bcx.ins().call(spill_ref, &[slot_arg, tag_arg, raw_arg]);
                    } else {
                        bcx.ins().call(update_raw_ref, &[slot_arg, raw_arg]);
                    }
                }
                // Call helper.
                let a_arg = bcx.ins().iconst(types::I64, a_us as i64);
                let n_arg = bcx.ins().iconst(types::I64, n_operands as i64);
                let func_ref = module.declare_func_in_func(op_concat_id, bcx.func);
                let call_inst = bcx.ins().call(func_ref, &[a_arg, n_arg]);
                let status = bcx.inst_results(call_inst)[0];
                let zero = bcx.ins().iconst(types::I64, 0);
                let is_err = bcx.ins().icmp(IntCC::SignedLessThan, status, zero);
                let continue_blk = bcx.create_block();
                let deopt_blk = bcx.create_block();
                bcx.ins().brif(is_err, deopt_blk, &[], continue_blk, &[]);
                bcx.switch_to_block(deopt_blk);
                bcx.seal_block(deopt_blk);
                emit_store_back_and_return_pc(
                    &mut bcx,
                    &regs_full[..max_stack],
                    reg_state,
                    rop.pc,
                    flush_ctx.as_ref(),
                    0i64,
                    trace_fn_sig_ref,
                    encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                );
                bcx.switch_to_block(continue_blk);
                bcx.seal_block(continue_blk);
                // Reload regs[A] (= result Str) from vm.stack via
                // luna_jit_stack_load helper. current_kinds = Unset
                // since RegKind doesn't carry Str (string raw bits
                // round-trip via the dispatcher's per-slot tag path).
                let stack_load_ref = module.declare_func_in_func(stack_load_id, bcx.func);
                let a_arg_reload = bcx.ins().iconst(types::I64, a_us as i64);
                let reload_inst = bcx.ins().call(stack_load_ref, &[a_arg_reload]);
                let result_raw = bcx.inst_results(reload_inst)[0];
                bcx.def_var(regs[a_us], result_raw);
                current_kinds[off + a_us] = RegKind::Unset;
            }
            // P12-S12-B-v2 — generic-for prep is the leading pc-bump
            // before body_top. Recorder enters at body_top, so this
            // arm is defensive only — pre-emit pass bails before we
            // reach it.
            Op::TForPrep => unreachable!("Op::TForPrep bailed in pre-emit pass"),
            // P12-S12-B-v2 — TForLoop is the trace's terminator; tail
            // emit handles the side-exit + back-edge.
            Op::TForLoop => unreachable!("Op::TForLoop only appears at effective_end"),
            _ => unreachable!("non-whitelisted op rejected in pre-emit pass"),
        }
    }

    // --- tail.
    //
    // Four cases pick the clean-close shape:
    //
    // - Trace truncated by `Op::Call` → store back + return
    //   `call.pc`. The Call's interp re-execution is the
    //   "exit" — no loop possible.
    // - Trace closes on `Op::ForLoop` (5.4+ Int count form) →
    //   emit the count check + step IR, then either jump back
    //   to `body_loop` (continue path) or side-exit at
    //   `forloop.pc + 1` (loop exit). In one-shot mode the
    //   continue path returns `head_pc` instead, so the
    //   dispatcher gets one iter per entry.
    // - `opts.internal_loop && has_cmp` (and no Call truncation,
    //   no ForLoop) → jump back to `body_loop`. The trace runs
    //   natively until some cmp side-exits; the dispatcher's
    //   per-entry marshal cost amortizes across however many
    //   iterations the loop runs.
    // - Otherwise (one-shot mode or a no-cmp trace) → store back
    //   + return `head_pc`. The dispatcher re-enters per
    //   iteration; an internal loop with no side-exit would
    //   spin forever.
    let has_cmp = record.ops[..effective_end]
        .iter()
        .any(|r| matches!(r.inst.op(), Op::Lt | Op::Le | Op::Eq));
    // P16-B — SelfLink close ALSO permits internal loop (in fact it
    // REQUIRES it — the whole point of the trace is to loop with
    // bump-base + branch-to-self). Treat self_link_idx_opt the same as
    // the cmp/ForLoop loop-permission predicates, and (critically)
    // don't trip on inline_abort_idx_opt — the SelfLink close path
    // doesn't go through that arm.
    // v2.0 Track-R R3a — `downrec_idx_opt` does NOT permit internal
    // loop (R3a routes through the R1 safe deopt path, single-shot).
    // R3b's lift to a real native back-edge will introduce its own
    // tail shape; until then, treat DownRec the same as a hard
    // truncation marker that forces one-shot dispatch.
    let do_internal_loop = opts.internal_loop
        && (has_cmp || for_loop_idx_opt.is_some() || self_link_idx_opt.is_some())
        && call_idx_opt.is_none()
        && return_idx_opt.is_none()
        && inline_abort_idx_opt.is_none()
        && downrec_idx_opt.is_none();
    // P12-S4-step3b — every tail `emit_store_back_and_return_pc`
    // passes `&regs_full[..max_stack]` so the store-back ONLY writes
    // the caller's window back to interp stack. Slots at
    // [max_stack..window_size) are inline-frame scratch and must not
    // leak into the dispatcher's reg_state restore.
    let caller_regs: &[Variable] = &regs_full[..max_stack];
    // v2.0 Track-R R3b — populated by the `downrec_idx_opt` arm when
    // it emits the stitch sentinel. Flows into `CompiledTrace.
    // downrec_link` at the struct literal below. `None` for every
    // other close shape.
    let mut downrec_link_for_compiled: Option<(u32, u32)> = None;
    let mut downrec_multi_way_count_for_compiled: u8 = 0;
    if let Some((_dr_idx, dr_return_pc, _target_proto_id, _depth_delta)) = downrec_idx_opt {
        // v2.0 Track-R R3b — `TraceEnd::DownRec` close: emit the
        // stitch-sentinel + caller-pc-guard scaffold.
        //
        // Shape mirrors LuaJIT's `asm_retf` (`lj_asm_arm64.h:565`):
        //   1. Load saved caller PC stand-in.
        //   2. CMP against IR-baked `dr_return_pc`.
        //   3. brif eq → stitch_blk: return DOWNREC sentinel +
        //      `record.head_pc` so the dispatcher (R3c) can walk
        //      `downrec_link` + RetfRecord chain to materialise the
        //      inlined frame and tail-call into the stitched child
        //      trace.
        //   4. brif ne → deopt_blk: R1's safe deopt-tail — store
        //      back caller window + return `head_pc` through the
        //      GLOBAL sentinel; the dispatcher resumes interp at
        //      head_pc with the trace's `dispatchable = false` gate
        //      blocking re-entry.
        //
        // Why the guard's load operand is `iconst(0)` today (NOT a
        // real `[reg_state + reserved_slot * 8]` load): luna's
        // trace ABI is `fn(reg_state: *mut i64) -> i64` — there is
        // no slot in the current `reg_state` buffer that the
        // dispatcher populates with the runtime saved caller PC.
        // Wiring that slot is R3c's job (it must touch the
        // dispatcher pre-trace-invoke path in `exec.rs` to write the
        // saved PC into a reserved position). Until then, the
        // immediate `iconst(0)` makes the guard's runtime
        // comparison `0 == dr_return_pc`; cranelift constant-folds
        // this to "always false" at codegen time (`dr_return_pc !=
        // 0` for every valid recording — Op::Return's PC is past
        // the prologue), so the emitted machine code unconditionally
        // jumps to `deopt_blk`. Net behaviour identical to R3a's
        // safe fall-through; both R3b's `downrec_link = Some(_)`
        // scaffold AND the safe deopt land in this commit, ready
        // for R3c to swap the `iconst(0)` for a real saved-PC load
        // once the dispatcher exposes the slot.
        //
        // `_target_proto_id` / `_depth_delta` consumed in IR by R3c
        // (target_proto_id becomes the helper-call argument that
        // walks `parent_ct.downrec_close.target_proto`; depth_delta
        // tunes how many CallFrames to push). R3b leaves them
        // touched as `let _ = ...` to keep rustc seeing the fields
        // as live for the next sub-step's hookup.
        debug_assert!(
            dr_return_pc != 0,
            "DownRec recorder should never trip on a PC=0 Op::Return — Op::Return's PC is past the prologue"
        );
        let _ = _target_proto_id;
        let _ = _depth_delta;

        let stitch_blk = bcx.create_block();
        let deopt_blk = bcx.create_block();

        // v2.0 Track-R R3d — multi-way caller-pc guard. R3c shipped a
        // single CMP (`saved_pc == dr_return_pc`) which measured a
        // 90% miss-rate on fib(3) hot-loop (R3c verdict §3) because
        // the typical fib body has TWO call sites at distinct
        // `pc + 1` caller_pcs — only one of them ever matched
        // `dr_return_pc` (the recorder picks the most-recent
        // threshold-tripping one). The recorder's `rec.retfs`
        // side-channel already collected every depth>0 Return's
        // `caller_pc` + `proto`, so the lowerer here can fan the
        // single CMP into a chain of `icmp(Equal, saved_pc, iconst
        // (candidate_pc)) + brif(eq, stitch, next)` predicates and
        // accept any of them as a HIT. Dedupe over `caller_pc`
        // (mirrors LuaJIT `lj_record.c:897 check_downrec_unroll`'s
        // "count IR_RETF entries by op1 == ptref" walk filtered to
        // the close marker's `target_proto`).
        //
        // Saved-PC slot (`reg_state[window_size_us * 8]`) populated
        // by R3c's dispatcher pre-invoke (see `crates/luna-core/src/
        // vm/exec.rs` `is_downrec_entry` block) with the parent
        // (caller) frame's `pc` — the runtime analogue of LuaJIT's
        // `[base-8]` in `asm_retf` (`lj_asm_arm64.h:565`).
        let saved_pc_offset = (window_size_us as i32) * 8;
        let saved_pc = bcx
            .ins()
            .load(types::I64, MemFlags::trusted(), reg_state, saved_pc_offset);
        // Collect distinct caller_pcs from retfs whose proto matches
        // the close marker's `_target_proto_id`. Dedupe + bound to
        // `DOWNREC_MULTI_WAY_GUARD_MAX` so IR size stays predictable
        // regardless of how many retfs the recorder captured.
        // `dr_return_pc` (the close marker's most-recent caller_pc) is
        // inserted first so the chain covers the single-CMP shape's
        // baseline even when filtering eliminates it.
        let mut candidates: Vec<u32> = Vec::with_capacity(DOWNREC_MULTI_WAY_GUARD_MAX);
        candidates.push(dr_return_pc);
        for retf in &record.retfs {
            if candidates.len() >= DOWNREC_MULTI_WAY_GUARD_MAX {
                break;
            }
            if retf.proto.as_ptr() as usize == _target_proto_id
                && !candidates.contains(&retf.caller_pc)
            {
                candidates.push(retf.caller_pc);
            }
        }
        // Emit CMP-chain. For each candidate: `icmp(Equal, ...) + brif`.
        // The last candidate's miss arm branches directly to deopt_blk;
        // earlier candidates' miss arms branch into a fresh block that
        // becomes the next CMP's "current block".
        for (i, candidate_pc) in candidates.iter().enumerate() {
            let imm_pc = bcx.ins().iconst(types::I64, *candidate_pc as i64);
            let eq = bcx.ins().icmp(IntCC::Equal, saved_pc, imm_pc);
            let miss_blk = if i + 1 < candidates.len() {
                bcx.create_block()
            } else {
                deopt_blk
            };
            bcx.ins().brif(eq, stitch_blk, &[], miss_blk, &[]);
            if i + 1 < candidates.len() {
                bcx.switch_to_block(miss_blk);
                bcx.seal_block(miss_blk);
            }
        }
        let multi_way_candidate_count = candidates.len();

        // Hit: return DOWNREC sentinel + `record.head_pc` as the
        // low 32 bits. The full encoded value is
        //   raw_ret = (1u64 << 63)             // side-trace marker
        //           | ((DOWNREC_CODE as u64) << 56)
        //           | (record.head_pc as u64)
        // (bit 63 set so the dispatcher's `from_side_trace` branch
        // at `exec.rs:6354+` decodes through the sentinel switch).
        // R3c's stitch arm reads `parent_ct.downrec_link` for the
        // stitch target rather than looking up via `side_trace_cache`.
        bcx.switch_to_block(stitch_blk);
        bcx.seal_block(stitch_blk);
        let raw_ret =
            (1u64 << 63) | ((SIDE_SENT_DOWNREC_CODE as u64) << 56) | (record.head_pc as u64);
        let stitch_ret = bcx.ins().iconst(types::I64, raw_ret as i64);
        bcx.ins().return_(&[stitch_ret]);

        // Miss: R1's safe deopt-tail (identical to R3a's emit).
        bcx.switch_to_block(deopt_blk);
        bcx.seal_block(deopt_blk);
        emit_store_back_and_return_pc(
            &mut bcx,
            caller_regs,
            reg_state,
            record.head_pc,
            flush_ctx.as_ref(),
            0i64,
            trace_fn_sig_ref,
            encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
        );

        // R3b populates downrec_link with the placeholder
        // (trace_id=0, target_head_pc=record.head_pc). The
        // `trace_id=0` sentinel means "self-stitch — target is the
        // trace currently dispatching"; R3c interprets this when
        // resolving the stitch target.
        downrec_link_for_compiled = Some((0, record.head_pc));

        // v2.0 Track-R R3d — lift `dispatchable = true` when the
        // multi-way guard collected at least 2 distinct caller_pc
        // candidates. The single-CMP fallback (count == 1) keeps
        // R3c's `dispatchable = false` + `"downrec-stitch-pending"`
        // pin because the 90% miss-rate measured at R3c verdict §3
        // would translate to 90% extra deopt cost if the primary
        // dispatcher arm admitted the trace unconditionally. The
        // dispatcher's `is_downrec_entry` arm (see `crates/luna-core/
        // src/vm/exec.rs`) keys on `ct.downrec_link.is_some()` so
        // the saved-PC slot is populated + post-invoke classifier is
        // routed for BOTH the lifted (dispatchable=true) and
        // unlifted (dispatchable=false) cases — only the find
        // predicate's admit arm differs.
        //
        // `"downrec-stitch-lifted"` is a new close-cause label that
        // mirrors R3b's `"downrec-stitch-pending"` for the lifted
        // case (so probes can tally "how many traces took which
        // R3d branch" via `trace_close_cause_counts`). The
        // `dispatch_off_reason` only sets in the unlifted branch
        // because `dispatchable=true` traces have no `dispatch_off`
        // by definition.
        if multi_way_candidate_count >= 2 {
            dispatchable = true;
            // Surface the lifted shape via a dedicated counter
            // `multi_way_guard_emitted` (bumped at the close handler
            // in `crates/luna-core/src/vm/exec.rs` reading
            // `downrec_multi_way_count_for_compiled` below) rather
            // than via the close-cause taxonomy — close causes mean
            // "trace didn't dispatch for reason X" and this branch
            // DOES dispatch.
        } else {
            dispatchable = false;
            dispatch_off_reason = dispatch_off_reason.or(Some("downrec-stitch-pending"));
        }
        downrec_multi_way_count_for_compiled =
            multi_way_candidate_count.min(u8::MAX as usize) as u8;
    } else if let Some((_self_link_idx, _kind)) = self_link_idx_opt {
        // v2.0 Track-R R1 — RETF-guards correctness primitive replaces
        // the previous P16-B snapshot-restore tail.
        //
        // The legacy P16-B emit was:
        //   1. Compute `bump_off` from the last captured Op::Call's
        //      `caller_offset + A + 1`.
        //   2. Slot-copy `regs_full[i] = regs_full[bump_off + i]` for
        //      `i in 0..max_stack` (deepest inlined frame → head frame).
        //   3. `jump(body_loop)` for a tight native back-edge.
        //
        // That mirrored LuaJIT's `asm_tail_link` (`lj_asm.c:2131`) only
        // syntactically. LuaJIT's pre-op snapshots distinguish each
        // frame's typed-slot mapping; luna's slot-copy assumes deepest
        // frame layout == head frame layout, which is sound for plain
        // tail-recursion but not for self-recursion through a
        // non-tail-call body (fib: `Lt → branch → Sub Call Sub Call Add
        // Return`, with depth-0 Sub writes polluting head-frame slots
        // BEFORE the recursive Call, plus a depth>0 base-case Return
        // whose deeper frame layout doesn't match head's). R0 measured
        // fib(28) returning 45 (vs 317_811) on the p16-on path.
        //
        // R1 swaps the slot-copy + back-edge for a clean deopt: store
        // back the caller window, return `head_pc`, and pin
        // `dispatchable = false`. The trace still compiles (cranelift
        // accepts a valid back-edge-free fn so the body's mcode and
        // window_size extension stay sound) but the dispatcher's
        // pre-invoke `dispatchable` check refuses to enter it, so
        // interp runs the recursion naturally and produces the correct
        // result on the p16-on path.
        //
        // The `RetfRecord` side-channel populated by the recorder
        // (exec.rs gate on `p16_self_link_enabled`) captures the
        // inlined-frame topology that R3's down-rec stitch will consume
        // to guard a real native back-edge. R1 is the correctness floor;
        // R3 lifts dispatchable back to true via the stitch.
        //
        // `window_size_us` extension above (line ~3350 `record.self_link
        // _kind.is_some()` arm) stays intact — body emit still writes
        // depth>0 slots into the extended buffer, the writes are simply
        // dead until R3 reads them via stitch.
        emit_store_back_and_return_pc(
            &mut bcx,
            caller_regs,
            reg_state,
            record.head_pc,
            flush_ctx.as_ref(),
            0i64,
            trace_fn_sig_ref,
            encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
        );
        dispatchable = false;
        dispatch_off_reason = dispatch_off_reason.or(Some("self-link-retf-r1"));
    } else if let Some(call_idx) = call_idx_opt {
        emit_store_back_and_return_pc(
            &mut bcx,
            caller_regs,
            reg_state,
            record.ops[call_idx].pc,
            flush_ctx.as_ref(),
            0i64,
            trace_fn_sig_ref,
            encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
        );
    } else if let Some(inline_abort_idx) = inline_abort_idx_opt {
        // P12-S4-step3b — InlineAbort: emit-up-to-i, then store back
        // + return record.ops[i].pc. Dispatchable is forced false
        // below (the interp can't resume at a depth>0 PC without the
        // CallFrames the trace inlined past — that's step 4).
        emit_store_back_and_return_pc(
            &mut bcx,
            caller_regs,
            reg_state,
            record.ops[inline_abort_idx].pc,
            flush_ctx.as_ref(),
            0i64,
            trace_fn_sig_ref,
            encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
        );
    } else if let Some(return_idx) = return_idx_opt {
        // P12-S4-step4b-C-2 — Return0/Return1 at depth=0: caller frame
        // unwinds. Same shape as Call truncation — store back caller
        // window + return the Return op's PC so the interp re-executes
        // it with the correct register state. Subject to the same
        // length-gate dispatchable check below.
        emit_store_back_and_return_pc(
            &mut bcx,
            caller_regs,
            reg_state,
            record.ops[return_idx].pc,
            flush_ctx.as_ref(),
            0i64,
            trace_fn_sig_ref,
            encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
        );
    } else if let Some(for_loop_idx) = for_loop_idx_opt {
        // 5.4+ Int count form (validated above; pre53 bails).
        //
        //   if R[A+1] > 0:
        //     R[A]     = R[A] + R[A+2]    (next loop var)
        //     R[A+1]   = R[A+1] - 1       (decrement count)
        //     R[A+3]   = R[A]              (visible loop var copy)
        //     // continue → back-edge (body_loop) or return head_pc
        //   else:
        //     // exit → side-exit at forloop.pc + 1
        //
        // ForLoop is only set at depth=0 (ForLoop@d>0 closes via
        // InlineAbort), so `regs_full[a]` directly addresses the
        // caller window — no offset.
        let rop = &record.ops[for_loop_idx];
        let a = rop.inst.a() as usize;
        match rop.inst.op() {
            Op::ForLoop => {
                let count = bcx.use_var(regs_full[a + 1]);
                let zero = bcx.ins().iconst(types::I64, 0);
                let cond = bcx.ins().icmp(IntCC::SignedGreaterThan, count, zero);

                let continue_blk = bcx.create_block();
                let exit_blk = bcx.create_block();
                bcx.ins().brif(cond, continue_blk, &[], exit_blk, &[]);

                // exit branch: side-exit at forloop.pc + 1.
                bcx.switch_to_block(exit_blk);
                bcx.seal_block(exit_blk);
                emit_store_back_and_return_pc(
                    &mut bcx,
                    caller_regs,
                    reg_state,
                    rop.pc + 1,
                    flush_ctx.as_ref(),
                    0i64,
                    trace_fn_sig_ref,
                    encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                );

                // continue branch: do the increment + decrement + back-edge.
                bcx.switch_to_block(continue_blk);
                bcx.seal_block(continue_blk);
                let cur = bcx.use_var(regs_full[a]);
                let step = bcx.use_var(regs_full[a + 2]);
                let next = bcx.ins().iadd(cur, step);
                bcx.def_var(regs_full[a], next);
                let one = bcx.ins().iconst(types::I64, 1);
                let count_new = bcx.ins().isub(count, one);
                bcx.def_var(regs_full[a + 1], count_new);
                bcx.def_var(regs_full[a + 3], next);
                if do_internal_loop {
                    bcx.ins().jump(body_loop, &[]);
                } else {
                    // ForLoop's continue branch jumps to the loop's
                    // BODY START (= (rop.pc + 1) - bx per OP_FORLOOP's
                    // backward jump encoding), not record.head_pc.
                    // For trace shapes whose head_pc == body_start
                    // (the usual back-edge trace), they're equal.
                    // For side traces whose head_pc lands on the
                    // ForLoop op itself (head_pc=rop.pc) instead of
                    // the back-edge target — e.g. an outer ForLoop
                    // that got recorded as a side trace from an inner
                    // loop exit — returning record.head_pc would
                    // re-enter the ForLoop op and double-advance the
                    // counter. Compute the body start explicitly.
                    // See docs/known-bugs/trace-jit-nested-loop-
                    // wrong-result.md §5-§7 for the diagnosis.
                    let body_pc = ((rop.pc as i32) + 1 - rop.inst.bx() as i32).max(0) as u32;
                    emit_store_back_and_return_pc(
                        &mut bcx,
                        caller_regs,
                        reg_state,
                        body_pc,
                        flush_ctx.as_ref(),
                        0i64,
                        trace_fn_sig_ref,
                        encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                    );
                }
            }
            Op::TForLoop => {
                // P12-S12-B-v2/v4 — generic-for back-edge:
                //
                //   tag = tforcall_tag_var  // v4: from TForCall's
                //                           //     batched helper
                //                           //     return value
                //   if tag == NIL:  side-exit at tforloop.pc + 1
                //   elif tag == INT: R[A+2]=R[A+4] + back-edge
                //   else: deopt (unsupported iter return kind)
                //
                // The Nil branch reuses the existing dispatcher
                // restore path; push a per_exit_kinds snapshot with
                // [A+4] = RegKind::Nil so the Nil side-exit repacks
                // correctly (entry's tag for A+4 was Int, so
                // dispatcher without override would restore as Int
                // — wrong for Nil).
                let tag = bcx.use_var(tforcall_tag_var);

                let nil_const = bcx
                    .ins()
                    .iconst(types::I64, luna_core::runtime::value::raw::NIL as i64);
                let int_const = bcx
                    .ins()
                    .iconst(types::I64, luna_core::runtime::value::raw::INT as i64);
                let is_nil = bcx.ins().icmp(IntCC::Equal, tag, nil_const);
                let nil_exit_blk = bcx.create_block();
                let not_nil_blk = bcx.create_block();
                bcx.ins().brif(is_nil, nil_exit_blk, &[], not_nil_blk, &[]);

                // Nil-exit branch: snapshot per_exit_kinds with [A+4]
                // = Nil, then store back + return tforloop.pc + 1.
                bcx.switch_to_block(nil_exit_blk);
                bcx.seal_block(nil_exit_blk);
                let mut nil_snapshot: Vec<RegKind> = current_kinds[..max_stack].to_vec();
                if a + 4 < nil_snapshot.len() {
                    nil_snapshot[a + 4] = RegKind::Nil;
                }
                let tag_side_box_2: Box<TCellPtr> = Box::new(TCellPtr::null());
                let _tag_side_cell_addr_2 = (&*tag_side_box_2) as *const TCellPtr as i64;
                let tag_side_local_2 = per_exit_kinds.len() as u32;
                per_exit_kinds.push((rop.pc + 1, nil_snapshot, tag_side_box_2));
                emit_store_back_and_return_pc(
                    &mut bcx,
                    caller_regs,
                    reg_state,
                    rop.pc + 1,
                    flush_ctx.as_ref(),
                    0i64,
                    trace_fn_sig_ref,
                    encode_side_sentinel(SIDE_SENT_KIND_TAG, tag_side_local_2),
                );

                // Non-Nil branch: check it's Int (v2 only handles
                // Int-key iters like ipairs / numeric pairs).
                bcx.switch_to_block(not_nil_blk);
                bcx.seal_block(not_nil_blk);
                let is_int = bcx.ins().icmp(IntCC::Equal, tag, int_const);
                let continue_blk = bcx.create_block();
                let deopt_blk = bcx.create_block();
                bcx.ins().brif(is_int, continue_blk, &[], deopt_blk, &[]);

                // Deopt: unsupported iter return kind. Store back +
                // return TForLoop.pc so the interp re-executes the
                // back-edge in the slow path.
                bcx.switch_to_block(deopt_blk);
                bcx.seal_block(deopt_blk);
                emit_store_back_and_return_pc(
                    &mut bcx,
                    caller_regs,
                    reg_state,
                    rop.pc,
                    flush_ctx.as_ref(),
                    0i64,
                    trace_fn_sig_ref,
                    encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                );

                // Continue: R[A+2] = R[A+4] (ctrl writeback) +
                // back-edge / store_back+head_pc.
                bcx.switch_to_block(continue_blk);
                bcx.seal_block(continue_blk);
                let ctrl = bcx.use_var(regs_full[a + 4]);
                bcx.def_var(regs_full[a + 2], ctrl);
                if do_internal_loop {
                    bcx.ins().jump(body_loop, &[]);
                } else {
                    emit_store_back_and_return_pc(
                        &mut bcx,
                        caller_regs,
                        reg_state,
                        record.head_pc,
                        flush_ctx.as_ref(),
                        0i64,
                        trace_fn_sig_ref,
                        encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
                    );
                }
            }
            _ => unreachable!("for_loop_idx_opt only set for Op::ForLoop / Op::TForLoop"),
        }
    } else if do_internal_loop {
        bcx.ins().jump(body_loop, &[]);
    } else {
        emit_store_back_and_return_pc(
            &mut bcx,
            caller_regs,
            reg_state,
            record.head_pc,
            flush_ctx.as_ref(),
            0i64,
            trace_fn_sig_ref,
            encode_side_sentinel(SIDE_SENT_KIND_GLOBAL, 0),
        );
    }
    // Seal the loop head now that both predecessors are emitted
    // (entry → body_loop in the prelude; tail → body_loop from
    // whichever block we ended up in for the clean-close case
    // when internal loop is on).
    bcx.seal_block(body_loop);

    bcx.finalize();
    // Path C — `LUNA_TRACE_IR_DUMP=1` dumps the cranelift IR of every
    // compiled trace fn to stderr. Categorization + density-reduction
    // tool for layer-6 attribution (per-call IR op count is the gap).
    if std::env::var("LUNA_TRACE_IR_DUMP")
        .map(|v| v == "1")
        .unwrap_or(false)
    {
        eprintln!(
            "=== TRACE IR DUMP head_pc={} n_recorded_ops={} ===\n{}\n=== END ===",
            record.head_pc,
            record.ops.len(),
            ctx.func.display()
        );
    }
    // v2.1 Phase 1I.D — `LUNA_TRACE_ASM_DUMP=1` requests cranelift to
    // emit the post-regalloc machine-code disassembly (vcode) and dumps
    // it to stderr after `define_function`. Used for the cargo-asm
    // decomposition of the table-field IC under env-OFF vs env-ON.
    let want_asm_dump = std::env::var("LUNA_TRACE_ASM_DUMP")
        .map(|v| v == "1")
        .unwrap_or(false);
    if want_asm_dump {
        ctx.set_disasm(true);
    }
    module.define_function(fn_id, &mut ctx).ok()?;
    if want_asm_dump
        && let Some(cc) = ctx.compiled_code()
        && let Some(vcode) = cc.vcode.as_ref()
    {
        eprintln!(
            "=== TRACE ASM DUMP head_pc={} n_recorded_ops={} ===\n{}\n=== END ===",
            record.head_pc,
            record.ops.len(),
            vcode
        );
    }
    module.clear_context(&mut ctx);
    // v1.3 Phase AOT Stage 3 — module finalization is the JIT-specific
    // wrapper's job (see [`try_compile_trace_with_options`]). The
    // generic body emits the function definition and stops at
    // `clear_context`; the JIT wrapper calls `finalize_definitions`
    // + `get_finalized_function`, patches `compiled.entry` with the
    // real fn pointer, and parks the module on the Vm's
    // `storage.trace_handles` Vec.
    // The AOT pipeline (luna-aot) calls `ObjectModule::finish` /
    // `ObjectProduct::emit` to produce a `.o` file instead, and
    // resolves the trace symbol at static link time.

    // Op::ForLoop at the tail writes R[A] (next loop var), R[A+1]
    // (decremented count), and R[A+3] (visible loop var copy) —
    // all Int per the 5.4+ count form. Op::TForLoop writes R[A+2]
    // = R[A+4] on continue (TForLoop tail emit; R[A+4] = Int gated
    // by the tag check).
    if let Some(for_loop_idx) = for_loop_idx_opt {
        let rop = &record.ops[for_loop_idx];
        let a = rop.inst.a() as usize;
        match rop.inst.op() {
            Op::ForLoop => {
                current_kinds[a] = RegKind::Int;
                current_kinds[a + 1] = RegKind::Int;
                current_kinds[a + 3] = RegKind::Int;
            }
            Op::TForLoop => {
                current_kinds[a + 2] = RegKind::Int;
            }
            _ => {}
        }
    }
    // Derive exit_tags from the kind tracker's final state. Slots
    // the trace never touched stay `Untouched` (dispatcher restores
    // the entry tag); slots the trace wrote take the writer's
    // determined kind. The legacy `MoveFrom` variant isn't produced
    // by this pass — `current_kinds` propagates source kinds at the
    // Move op so the dispatcher doesn't need the deferred entry-tag
    // lookup.
    // P12-S4-step2c — dispatch heuristic: a `Op::Call`-truncated
    // trace whose body is too short to amortise the dispatcher's
    // marshal-in + transmute + restore overhead is a net loss vs the
    // interpreter (measured at ~1.8× slower on fib_28's ~7-op body,
    // even after the Rc<[]> exit_tags fix). Keep such traces cached
    // (compile cost is paid) but pin dispatchable=false until
    // step 3's inline emit makes the per-dispatch body large enough
    // to win. `MIN_DISPATCHABLE_TRUNC_BODY_BASE` is tuned to fib's
    // 7-op body being just below the gate at depth=0.
    //
    // P13-S13-C — scale the gate down as `max_depth_used` grows:
    // each extra inline level amortises ~2 ops worth of marshal
    // overhead per dispatch (one dispatch processes the full
    // chain of depth+1 frames). Saturating-sub so deep traces
    // never miss-fire on the length gate.
    const MIN_DISPATCHABLE_TRUNC_BODY_BASE: usize = 20;
    // P13-S13-G v3 — floor at 40 ops/dispatch (the empirical
    // dispatcher-overhead amortisation line: ~80ns per dispatch /
    // ~2ns per body op). S13-C's adaptive `BASE - depth*2`
    // formula could drop the gate to 0 at MAX_INLINE_DEPTH=16,
    // letting tiny-body inline traces dispatch and pay overhead
    // they can't amortise (binary_trees_d4 0.73× regression
    // — see `docs/rfcs/20260622-p13-s13g-cross-proto-call/design.md`
    // Cause C). The floor doesn't affect fib_28 (~112 ops body
    // post-S13-C — well above the floor) but bails the
    // binary_trees pathological case.
    const MIN_DISPATCHABLE_TRUNC_BODY_FLOOR: usize = 40;
    let max_depth_used = record
        .ops
        .iter()
        .map(|r| r.inline_depth as usize)
        .max()
        .unwrap_or(0);
    let adaptive = MIN_DISPATCHABLE_TRUNC_BODY_BASE.saturating_sub(max_depth_used * 2);
    let min_dispatchable_trunc_body = adaptive.max(MIN_DISPATCHABLE_TRUNC_BODY_FLOOR);
    // P12-S4-step4b-C-2 — inline traces (per_exit_metas non-empty)
    // skip the length-gate. Each dispatch tears through multiple
    // inlined frames so body-length isn't a useful proxy for the
    // dispatcher's marshal overhead; the gate would dump fib's
    // ~8-op-by-the-time-MAX_DEPTH-hits prefix even though one
    // dispatch processes 4 recursion levels.
    //
    // P12-S5-B — sunk-alloc traces also skip the length-gate.
    // Skipping even a single `Heap::new_table()` per dispatch
    // dwarfs the marshal-in/out overhead on a 7-op body, so the
    // gate's conservative default is a net loss here.
    //
    // P12-S7-A NOTE: closure-creating traces do NOT skip the
    // length-gate. Unlike sunk emit which avoids `Heap::new_table()`,
    // the Op::Closure helper still calls `Heap::new_closure_inline`
    // — emit replaces only the interp's match-arm dispatch +
    // frame plumbing for the 2-op `Closure + Return1` shape, which
    // is less than trace dispatch's marshal+enter overhead. Per-iter
    // dispatch of a tiny closure-constructor body is a net loss
    // (probe: `closure_no_upval_for_500k` mac measured 0.53× when
    // the gate was skipped). Closure traces only earn dispatch when
    // body length passes the gate organically.
    if (call_idx_opt.is_some() || return_idx_opt.is_some())
        && effective_end < min_dispatchable_trunc_body
        && per_exit_inline_vec.is_empty()
        && sunk_alloc_seen == 0
    {
        dispatchable = false;
        dispatch_off_reason = dispatch_off_reason.or(Some("length-gate"));
    }
    // P12-S4-step3b — InlineAbort traces close before any frame
    // materialization machinery exists (step 4's job). The interp
    // can't resume at the inline-abort PC without the matching
    // CallFrames; gate dispatch off until step 4 adds the helper.
    if inline_abort_idx_opt.is_some() {
        dispatchable = false;
        dispatch_off_reason = dispatch_off_reason.or(Some("InlineAbort-gate"));
    }

    // P12-S4-step3b — clean-tail `exit_tags` cover the caller window
    // only ([0..max_stack)). Per-side-exit `per_exit_tags` for inline
    // cmp sites (step4b-C-2) carry the full `window_size` snapshot
    // because the dispatcher must restore EVERY pushed frame's
    // register window, not just the caller's.
    let mut exit_tags_vec = kinds_to_exit_tags(&current_kinds[..max_stack]);
    // P12-S5-B — for every sunk site at depth=0 (depth>0 is rejected
    // in pre-emit), force the slot's exit tag to `Untouched` so the
    // dispatcher carries the entry tag in the restore. Without this
    // override the slot's `current_kinds` could read as Table (from
    // some other path) or Unset, and the dispatcher would try to
    // unpack `reg_state[a]` (which we never wrote for sunk sites)
    // as a `Value::Table` of NULL bits → SIGSEGV.
    for site in &escape.sites {
        if site.state == EscapeState::Sinkable && site.inline_depth == 0 {
            let idx = site.a as usize;
            if idx < exit_tags_vec.len() {
                exit_tags_vec[idx] = ExitTag::Untouched;
            }
        }
    }
    let global_tag_res_kind = classify_exit_tags(&exit_tags_vec);
    let exit_tags: TArc<[ExitTag]> = exit_tags_vec.into();
    // P15-A v2-C-A2 — split per_exit_kinds's 3-tuple into the
    // 2-tuple `per_exit_tags` for the dispatcher AND the parallel
    // `tags_side_trace_ptrs` Box slice the close handler writes to.
    // The Box transports the cell's heap address (baked into the
    // IR's `iconst` at each callsite) through this move without
    // moving the cell itself.
    let mut tags_side_boxes: Vec<Box<TCellPtr>> = Vec::with_capacity(per_exit_kinds.len());
    let per_exit_tags: TArc<[(u32, TArc<[ExitTag]>)]> = per_exit_kinds
        .into_iter()
        .map(|(pc, kinds, side_box)| {
            // step4b-C-2 — the cmp emit site pushed the right slice
            // length (caller-window for depth=0, full window for
            // depth>0). Hand it through verbatim — the dispatcher
            // iterates `exit_tags_for_pc.len()` and walks both
            // shapes uniformly.
            let tags: TArc<[ExitTag]> = kinds_to_exit_tags(&kinds).into();
            tags_side_boxes.push(side_box);
            (pc, tags)
        })
        .collect::<Vec<_>>()
        .into();
    let tags_side_trace_ptrs: TArc<[Box<TCellPtr>]> = tags_side_boxes.into();
    let per_exit_inline: TArc<[InlineSideExit]> = per_exit_inline_vec
        .into_iter()
        .map(
            |(cont_pc, head_resume_pc, kinds, chain, side_trace_ptr)| InlineSideExit {
                cont_pc,
                head_resume_pc,
                exit_tags: kinds_to_exit_tags(&kinds).into(),
                chain,
                side_trace_ptr,
            },
        )
        .collect::<Vec<_>>()
        .into();

    checkpoint("post:emit-pass-done");
    // P15-prep — pre-compute exit_hit_counts before the struct
    // init so per_exit_tags's len is still accessible.
    let exit_hit_counts: TArc<[TCellU32]> = {
        let total = per_exit_inline.len() + per_exit_tags.len() + 1;
        let v: Vec<TCellU32> = (0..total).map(|_| TCellU32::new(0)).collect();
        v.into()
    };
    // P15-A v2-A — parallel per-exit raw fn-ptr slots, all null
    // until a child side trace compiles for the slot. Same length
    // as exit_hit_counts; v2-B/C will read these from IR.
    let exit_side_trace_ptrs: TArc<[TCellPtr]> = {
        let total = per_exit_inline.len() + per_exit_tags.len() + 1;
        let v: Vec<TCellPtr> = (0..total).map(|_| TCellPtr::null()).collect();
        v.into()
    };
    let compiled = CompiledTrace {
        head_pc: record.head_pc,
        // v1.3 Phase AOT Stage 3 — caller (JIT wrapper or AOT pipeline)
        // patches `entry` after finalize. See [`placeholder_trace_fn`].
        entry: placeholder_trace_fn,
        n_ops: record.ops.len() as u32,
        dispatchable,
        // P12-S4-step3b — real window_size now ≥ max_stack; the
        // dispatcher reads this to size its reg_state buffer.
        window_size,
        exit_tags,
        global_tag_res_kind,
        is_inline_abort_close: inline_abort_idx_opt.is_some(),
        dispatch_off_reason: if dispatchable {
            None
        } else {
            dispatch_off_reason
        },
        entry_tags: record.entry_tags.clone().into(),
        per_exit_tags,
        // P12-S4-step4b-C-2 — populated by the cmp@d>0 emit sites
        // above; the IR encodes `(site_idx + 1)` in the upper 32
        // bits of its return value so the dispatcher can pull the
        // right entry. Holding the inner Rc<[FrameMaterializeInfo]>
        // alive keeps each chain's address stable across dispatches
        // (cranelift IR has the raw pointer baked in via iconst).
        exit_hit_counts,
        exit_side_trace_ptrs,
        // P15-A v2-C-A2 — per-TAG-entry side-trace cells (parallel
        // to per_exit_tags) + the GLOBAL singleton cell. Both
        // collected from Boxes allocated AT each emit callsite so
        // the IR has baked the right heap address.
        tags_side_trace_ptrs,
        global_side_trace_ptr: global_side_trace_box,
        // P15-A v2-C-A1 — empty at compile; close handler fills
        // it as child side traces compile for this trace's hot
        // exits.
        side_trace_cache: TRefLock::new(std::collections::HashMap::new()),
        has_any_side_wired: TCellBool::new(false),
        per_exit_inline,
        // P12-S5-A — diagnostic only; counts Sinkable sites from the
        // pre-emit sweep. Vm sums these into `trace_sinkable_seen_count`
        // for sprint-level visibility.
        sinkable_sites_seen: escape.sinkable_count(),
        accum_bufferable_seen: escape
            .accum_sites
            .iter()
            .filter(|s| s.state == BufferState::Bufferable)
            .count() as u32,
        // P12-S5-B — count of sites that actually took the sunk-emit
        // path in this trace's body (NewTable replaced by virt slot
        // Variables, no heap alloc helper called). Vm bumps
        // `trace_sunk_alloc_count` by this on compile success.
        sunk_alloc_seen,
        // P12-S5-C — count of materialise emit sites for sunk slot
        // recovery at cmp side-exits.
        materialize_emit_count,
        // P12-S7-A — count of Op::Closure ops the trace lowered.
        closure_seen,
        // P15-A v2-E — compute body_writes for the smart side-trace
        // gate. Uses op_offsets (already computed above) to apply
        // inline-depth offsets per op.
        body_writes: compute_body_writes(record, &op_offsets).into(),
        // v2.0 Track-R R3b — populated by the `downrec_idx_opt` arm
        // above into `downrec_link_for_compiled`. When the arm
        // emitted a stitch sentinel + caller-pc guard, this carries
        // `Some((0, head_pc))`; otherwise `None`. R3b deliberately
        // keeps `dispatchable = false` even when `Some(_)` — R3d
        // lifts to `dispatchable = true` when the multi-way candidate
        // count >= 2 (see `downrec_multi_way_count` below).
        downrec_link: downrec_link_for_compiled,
        // v2.0 Track-R R3d — multi-way guard candidate count baked
        // into the IR's CMP-chain. `0` for non-DownRec closes;
        // `1` for single-CMP-fallback DownRec; `>= 2` for the
        // lifted `dispatchable = true` path.
        downrec_multi_way_count: downrec_multi_way_count_for_compiled,
    };
    Some((fn_id, compiled))
}

#[cfg(test)]
mod s14b_v0_scaffolding {
    //! P14-S14-B v0 — surface tests for the accumulator-detection
    //! scaffolding. v0 is a stub: the API shape is committed but
    //! the detector returns an empty Vec. These tests pin the
    //! surface so v1+ code can extend without API churn.
    use super::{AccumSite, BufferState, EscapeAnalysis};

    #[test]
    fn buffer_state_variants_exist() {
        let b = BufferState::Bufferable;
        let nb = BufferState::NonBuffered;
        assert_ne!(b, nb);
    }

    #[test]
    fn accum_site_clones_cleanly() {
        let site = AccumSite {
            op_idx: 0,
            pc: 0,
            accum_slot: 0,
            piece_slot: 1,
            inline_depth: 0,
            state: BufferState::Bufferable,
        };
        let cloned = site.clone();
        assert_eq!(site.op_idx, cloned.op_idx);
        assert_eq!(site.state, cloned.state);
    }

    #[test]
    fn escape_analysis_default_has_empty_accum_fields() {
        let ea: EscapeAnalysis = Default::default();
        assert!(ea.accum_sites.is_empty());
        assert!(ea.accum_live_at_op.is_empty());
    }
}

#[cfg(test)]
mod s13a_depth_invariant {
    //! P13-S13-A — pure-function tests for `verify_depth_invariant`.
    //! Synthetic `(depth, is_call)` sequences exercise the depth
    //! contract without needing a `Gc<Proto>`.
    use super::{MAX_INLINE_DEPTH, verify_depth_invariant};

    #[test]
    fn empty_sequence_is_valid() {
        assert!(verify_depth_invariant(&[]));
    }

    #[test]
    fn single_op_at_depth_zero_is_valid() {
        assert!(verify_depth_invariant(&[(0, false)]));
        assert!(verify_depth_invariant(&[(0, true)]));
    }

    #[test]
    fn single_op_at_nonzero_depth_is_invalid() {
        assert!(!verify_depth_invariant(&[(1, false)]));
        assert!(!verify_depth_invariant(&[(2, true)]));
    }

    #[test]
    fn linear_ascent_one_step_at_a_time_is_valid() {
        // 0 (Call) → 1 (Call) → 2 (Call) → 3
        assert!(verify_depth_invariant(&[
            (0, true),
            (1, true),
            (2, true),
            (3, false),
        ]));
    }

    #[test]
    fn ascent_without_preceding_call_is_invalid() {
        // 0 (not Call) → 1 — violates "depth bump must follow Op::Call".
        assert!(!verify_depth_invariant(&[(0, false), (1, false)]));
    }

    #[test]
    fn ascent_skipping_a_depth_level_is_invalid() {
        // 0 → 2 (skip 1). Even with preceding Op::Call, the recorder
        // must surface every intermediate frame.
        assert!(!verify_depth_invariant(&[(0, true), (2, false)]));
    }

    #[test]
    fn arbitrary_descent_is_valid() {
        // Climb to depth 3, then drop straight to 0 (multi-Return).
        assert!(verify_depth_invariant(&[
            (0, true),
            (1, true),
            (2, true),
            (3, false),
            (0, false),
        ]));
    }

    #[test]
    fn re_ascent_after_descent_is_valid() {
        // 0 (Call) → 1 → 0 → 1 (Call) → 2. Each bump preceded by a
        // Call; descents are unconstrained.
        assert!(verify_depth_invariant(&[
            (0, true),
            (1, false),
            (0, true),
            (1, true),
            (2, false),
        ]));
    }

    #[test]
    fn boundary_at_max_inline_depth_is_valid() {
        // Walk all the way up to MAX_INLINE_DEPTH. Each bump preceded
        // by an Op::Call.
        let mut items: Vec<(u8, bool)> = Vec::new();
        for d in 0..=MAX_INLINE_DEPTH {
            // The op at depth d is an Op::Call iff there's a deeper
            // op to come (it'll push the next frame).
            let is_call = d < MAX_INLINE_DEPTH;
            items.push((d, is_call));
        }
        assert!(verify_depth_invariant(&items));
    }

    #[test]
    fn depth_exceeding_max_inline_depth_is_invalid() {
        // Hit MAX_INLINE_DEPTH + 1. Each step legal, but the cap is
        // exceeded.
        let mut items: Vec<(u8, bool)> = Vec::new();
        for d in 0..=MAX_INLINE_DEPTH {
            items.push((d, true));
        }
        items.push((MAX_INLINE_DEPTH + 1, false));
        assert!(!verify_depth_invariant(&items));
    }

    #[test]
    fn descent_then_ascent_without_call_is_invalid() {
        // 0 (Call) → 1 (not Call) → 0 → 1 — the second ascent's
        // previous op is the depth-0 op that's NOT a Call.
        assert!(!verify_depth_invariant(&[
            (0, true),
            (1, false),
            (0, false),
            (1, false),
        ]));
    }
}

#[cfg(test)]
mod s2b_arith {
    use super::*;
    use luna_core::version::LuaVersion;
    use luna_core::vm::Vm;
    use luna_core::vm::isa::{Inst, Op};

    /// Load a Lua chunk and return its outer Proto. The Vm must
    /// outlive the returned `Gc<Proto>` — `Heap::drop` frees every
    /// GC object regardless of reachability, so a `wide_proto()`
    /// helper that owns its own Vm would hand back a dangling
    /// pointer the moment it returns. Every test binds the Vm to
    /// a local and threads it explicitly.
    fn load_proto(vm: &mut Vm, src: &[u8]) -> Gc<Proto> {
        let cl = vm.load(src, b"=t").expect("compile");
        cl.proto
    }

    /// Chunk source sized for ≥ 4 regs (`max_stack` = 5) — enough
    /// for every step-2 test that touches R[0..=3].
    const WIDE_SRC: &[u8] = b"local a,b,c,d = 0,0,0,0; return a+b+c+d";

    fn make_record(head_pc: u32, ops: &[Inst], proto: Gc<Proto>) -> TraceRecord {
        let mut rec = TraceRecord::start(proto, head_pc, Vec::new(), false);
        for (i, inst) in ops.iter().copied().enumerate() {
            let pushed = rec.push(RecordedOp {
                proto,
                pc: i as u32,
                inst,
                inline_depth: 0,
                var_count: None,
            });
            assert!(pushed, "test trace must fit MAX_TRACE_LEN");
        }
        rec.closed = true;
        rec
    }

    #[test]
    fn closed_empty_trace_returns_head_pc_and_passes_regs_through() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let rec = make_record(7, &[], p);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec)
            .expect("empty closed trace must compile");
        assert_eq!(ct.head_pc, 7);
        assert_eq!(ct.n_ops, 0);

        let mut state: Vec<i64> = vec![100, 200, 300, 400];
        // Resize to head_proto.max_stack so the trace's full store-back
        // pass has somewhere to write. Extra slots default to 0.
        state.resize(p.max_stack as usize, 0);
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };

        assert_eq!(r, 7, "clean close returns head_pc");
        // First four slots passed through untouched — load-then-store
        // pattern preserves the i64 payload.
        assert_eq!(state[0], 100);
        assert_eq!(state[1], 200);
        assert_eq!(state[2], 300);
        assert_eq!(state[3], 400);
    }

    #[test]
    fn add_trace_computes_sum_into_dst_reg() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // R[0] = R[1] + R[2]
        let add = Inst::iabc(Op::Add, 0, 1, 2, false);
        let rec = make_record(11, &[add], p);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("Add trace must compile");
        assert_eq!(ct.head_pc, 11);
        assert_eq!(ct.n_ops, 1);

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[1] = 10;
        state[2] = 3;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };

        assert_eq!(r, 11);
        assert_eq!(state[0], 13, "10 + 3");
        assert_eq!(state[1], 10, "input untouched");
        assert_eq!(state[2], 3, "input untouched");
    }

    #[test]
    fn chained_arith_threads_results_through_regs() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // R[0] = R[0] + R[1]
        // R[0] = R[0] * R[2]
        // R[0] = R[0] - R[3]
        let prog = [
            Inst::iabc(Op::Add, 0, 0, 1, false),
            Inst::iabc(Op::Mul, 0, 0, 2, false),
            Inst::iabc(Op::Sub, 0, 0, 3, false),
        ];
        let rec = make_record(0, &prog, p);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec)
            .expect("Add/Mul/Sub chain must compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = 5;
        state[1] = 3;
        state[2] = 4;
        state[3] = 7;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };

        assert_eq!(r, 0, "head_pc 0");
        // ((5 + 3) * 4) - 7 = 25
        assert_eq!(state[0], 25);
    }

    #[test]
    fn move_then_mul_propagates_via_move() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // R[0] = R[3]      (Move)
        // R[0] = R[0] * R[1]
        let prog = [
            Inst::iabc(Op::Move, 0, 3, 0, false),
            Inst::iabc(Op::Mul, 0, 0, 1, false),
        ];
        let rec = make_record(0, &prog, p);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("Move + Mul must compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = 999; // overwritten by Move
        state[1] = 6;
        state[3] = 7;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };

        assert_eq!(r, 0);
        assert_eq!(state[0], 42, "7 * 6");
    }

    #[test]
    fn trailing_jmp_is_emit_no_op() {
        // The trace recorder typically pushes the back-edge Op::Jmp
        // as the last op before the close detection fires on the
        // next head_pc visit. Step 2 treats it as a no-op — the
        // tail (return head_pc) carries the control transfer.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [
            Inst::iabc(Op::Add, 0, 1, 2, false),
            // Use the unconditional Jmp shape; offset is irrelevant
            // to the lowerer (we know head_pc from the record).
            Inst::isj(Op::Jmp, -3),
        ];
        let rec = make_record(0, &prog, p);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec)
            .expect("Add + trailing Jmp must compile");
        assert_eq!(ct.n_ops, 2);

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[1] = 4;
        state[2] = 5;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        assert_eq!(r, 0);
        assert_eq!(state[0], 9);
    }

    #[test]
    fn non_closed_trace_does_not_compile() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let rec = TraceRecord::start(p, 0, Vec::new(), false);
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn unsupported_op_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // Op::Concat is not in the whitelist (and unlike Op::Return0
        // which step4b-C-2 promoted to a truncation point, Concat
        // has no special treatment and falls through to a bail).
        let prog = [Inst::iabc(Op::Concat, 0, 0, 0, false)];
        let rec = make_record(0, &prog, p);
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn inline_depth_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Add, 0, 1, 2, false),
            inline_depth: 1, // S4 territory — step 2 must bail.
            var_count: None,
        });
        rec.closed = true;
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn cross_proto_op_bails() {
        let mut vm1 = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let mut vm2 = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p1 = load_proto(&mut vm1, WIDE_SRC);
        let p2 = load_proto(&mut vm2, WIDE_SRC);
        // Distinct `Vm::new` runs → distinct Proto allocations,
        // even though both compile from the same source. The
        // lowerer must reject any cross-Proto op (inlined sub-calls
        // are S4 territory).
        let mut rec = TraceRecord::start(p1, 0, Vec::new(), false);
        rec.push(RecordedOp {
            proto: p2,
            pc: 0,
            inst: Inst::iabc(Op::Add, 0, 1, 2, false),
            inline_depth: 0,
            var_count: None,
        });
        rec.closed = true;
        assert!(try_compile_trace(vm1.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn out_of_bounds_reg_bails() {
        // `return 0` compiles to a tiny chunk with max_stack = 2
        // (proven by `cargo run --example probe_ms`); R[2] in an
        // arith op lies at the boundary (index 2 == len) and must
        // be rejected.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, b"return 0");
        let prog = [Inst::iabc(Op::Add, 0, 1, 2, false)];
        let rec = make_record(0, &prog, p);
        // Guard the assertion so a future compiler change to
        // small-chunk max_stack doesn't silently turn the test
        // green via the loose-precondition path.
        assert!((p.max_stack as usize) <= 2, "precondition for this test");
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn compiled_trace_is_callable_repeatedly() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [Inst::iabc(Op::Add, 0, 1, 2, false)];
        let rec = make_record(0, &prog, p);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        // Re-entrancy / mmap-lifetime sanity: each call recomputes
        // R[0] from the inputs, and the fn ptr stays valid because
        // `storage.trace_handles` keeps `JITModule` alive.
        for k in 0..1000 {
            state[1] = k;
            state[2] = 2 * k;
            let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
            assert_eq!(r, 0);
            assert_eq!(state[0], 3 * k);
        }
    }
}

#[cfg(test)]
mod s2b_cmp {
    use super::*;
    use luna_core::version::LuaVersion;
    use luna_core::vm::Vm;
    use luna_core::vm::isa::{Inst, Op};

    const WIDE_SRC: &[u8] = b"local a,b,c,d = 0,0,0,0; return a+b+c+d";

    fn load_proto(vm: &mut Vm, src: &[u8]) -> Gc<Proto> {
        let cl = vm.load(src, b"=t").expect("compile");
        cl.proto
    }

    /// Build a 2-op trace `[cmp, jmp_back]` of length 2. `cmp_pc`
    /// is what the cmp's `RecordedOp.pc` will be; the trailing Jmp
    /// gets `cmp_pc + 1` so it satisfies the "Jmp at cmp.pc + 1"
    /// rule. The trace's `head_pc` is supplied separately — it
    /// controls the clean-close return value and is independent of
    /// the cmp's PC.
    fn cmp_jmp_record(proto: Gc<Proto>, head_pc: u32, cmp_pc: u32, cmp: Inst) -> TraceRecord {
        let mut rec = TraceRecord::start(proto, head_pc, Vec::new(), false);
        rec.push(RecordedOp {
            proto,
            pc: cmp_pc,
            inst: cmp,
            inline_depth: 0,
            var_count: None,
        });
        rec.push(RecordedOp {
            proto,
            pc: cmp_pc + 1,
            inst: Inst::isj(Op::Jmp, -1),
            inline_depth: 0,
            var_count: None,
        });
        rec.closed = true;
        rec
    }

    #[test]
    fn lt_k1_returns_head_pc_when_cmp_matches() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // if (R[1] < R[2]) ~= 1 then pc++   — recorded "1 < 2", matched K=1.
        let lt = Inst::iabc(Op::Lt, 1, 2, 0, true);
        let rec = cmp_jmp_record(p, 5, 10, lt);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec)
            .expect("Lt + trailing Jmp must compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[1] = 3; // 3 < 7 holds → matches K=true → continue
        state[2] = 7;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        assert_eq!(r, 5, "clean close returns head_pc");
    }

    #[test]
    fn lt_k1_side_exits_when_cmp_inverts() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let lt = Inst::iabc(Op::Lt, 1, 2, 0, true); // K=1
        let rec = cmp_jmp_record(p, 5, 10, lt);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[1] = 9; // 9 < 7 false → mismatch with K=1 → side-exit
        state[2] = 7;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        // Lua's `pc++` on cmp mismatch lands at cmp_pc + 2 = 12.
        assert_eq!(r, 12, "side-exit returns failing PC = cmp_pc + 2");
        // Reg state is still written back so interp resumes
        // with consistent values.
        assert_eq!(state[1], 9);
        assert_eq!(state[2], 7);
    }

    #[test]
    fn lt_k0_inverts_continue_condition() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // if (R[1] < R[2]) ~= 0 then pc++  — recorded direction:
        // cmp result was false (9 < 7), matched K=0 → took Jmp.
        let lt = Inst::iabc(Op::Lt, 1, 2, 0, false);
        let rec = cmp_jmp_record(p, 3, 8, lt);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[1] = 9;
        state[2] = 7;
        // Cmp result `9 < 7` is false; K=0; false == K=0 → continue.
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        assert_eq!(r, 3);

        // Flip inputs so cmp result `3 < 7` is true; K=0; true != K → side-exit.
        state[1] = 3;
        state[2] = 7;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        assert_eq!(r, 10, "cmp_pc=8 + 2 = 10");
    }

    #[test]
    fn le_emits_signed_less_equal() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // if (R[1] <= R[2]) ~= 1 then pc++
        let le = Inst::iabc(Op::Le, 1, 2, 0, true);
        let rec = cmp_jmp_record(p, 0, 0, le);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        // Equal: 5 <= 5 holds → continue.
        state[1] = 5;
        state[2] = 5;
        assert_eq!(unsafe { (ct.entry)(state.as_mut_ptr()) }, 0);

        // 5 <= 4 false → side-exit.
        state[1] = 5;
        state[2] = 4;
        assert_eq!(unsafe { (ct.entry)(state.as_mut_ptr()) }, 2);
    }

    #[test]
    fn eq_emits_int_equality() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // if (R[0] == R[1]) ~= 1 then pc++
        let eq = Inst::iabc(Op::Eq, 0, 1, 0, true);
        let rec = cmp_jmp_record(p, 7, 4, eq);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = 42;
        state[1] = 42;
        assert_eq!(unsafe { (ct.entry)(state.as_mut_ptr()) }, 7);

        state[0] = 42;
        state[1] = 41;
        assert_eq!(unsafe { (ct.entry)(state.as_mut_ptr()) }, 6); // cmp_pc(4) + 2
    }

    #[test]
    fn arith_then_cmp_side_exit_stores_back_post_arith_value() {
        // Trace records: R[0] = R[0] - R[1]; Lt R[0] R[2] K=1; Jmp -3.
        // The side-exit must write the *post-arith* R[0] to reg_state
        // so interp resumes with the value the trace just computed.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [
            Inst::iabc(Op::Sub, 0, 0, 1, false),
            Inst::iabc(Op::Lt, 0, 2, 0, true), // R[0] < R[2]
            Inst::isj(Op::Jmp, -3),
        ];
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        for (i, inst) in prog.iter().copied().enumerate() {
            rec.push(RecordedOp {
                proto: p,
                pc: i as u32,
                inst,
                inline_depth: 0,
                var_count: None,
            });
        }
        rec.closed = true;
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        // R[0] = 10 - 7 = 3; 3 < 5 → continue → return head_pc.
        state[0] = 10;
        state[1] = 7;
        state[2] = 5;
        assert_eq!(unsafe { (ct.entry)(state.as_mut_ptr()) }, 0);
        assert_eq!(state[0], 3);

        // R[0] = 10 - 1 = 9; 9 < 5 false → side-exit.
        state[0] = 10;
        state[1] = 1;
        state[2] = 5;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        // cmp at index 1, pc=1; failing PC = pc+2 = 3.
        assert_eq!(r, 3);
        assert_eq!(state[0], 9, "post-arith value must be in reg_state");
        assert_eq!(state[1], 1);
        assert_eq!(state[2], 5);
    }

    #[test]
    fn cmp_at_trailing_position_bails() {
        // No Jmp follows — step 3 doesn't capture the
        // "cmp didn't match K, Jmp skipped" direction.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Lt, 0, 1, 0, true),
            inline_depth: 0,
            var_count: None,
        });
        rec.closed = true;
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn cmp_followed_by_non_jmp_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [
            Inst::iabc(Op::Lt, 0, 1, 0, true),
            Inst::iabc(Op::Add, 2, 0, 1, false), // not a Jmp
        ];
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        for (i, inst) in prog.iter().copied().enumerate() {
            rec.push(RecordedOp {
                proto: p,
                pc: i as u32,
                inst,
                inline_depth: 0,
                var_count: None,
            });
        }
        rec.closed = true;
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn cmp_jmp_with_wrong_pc_offset_bails() {
        // Jmp must be at cmp.pc + 1. A Jmp at cmp.pc + 5 is treated
        // as an orphan and bails.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Lt, 0, 1, 0, true),
            inline_depth: 0,
            var_count: None,
        });
        rec.push(RecordedOp {
            proto: p,
            pc: 5, // not 1
            inst: Inst::isj(Op::Jmp, -1),
            inline_depth: 0,
            var_count: None,
        });
        rec.closed = true;
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn orphan_jmp_mid_trace_bails() {
        // A Jmp that's not consumed by a cmp and not at the last
        // position must bail (step-3 doesn't know what to do with
        // a free-floating unconditional jump).
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [
            Inst::isj(Op::Jmp, -1), // orphan, position 0 of a 2-op record
            Inst::iabc(Op::Add, 0, 1, 2, false),
        ];
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        for (i, inst) in prog.iter().copied().enumerate() {
            rec.push(RecordedOp {
                proto: p,
                pc: i as u32,
                inst,
                inline_depth: 0,
                var_count: None,
            });
        }
        rec.closed = true;
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn loop_pattern_alternates_continue_and_exit() {
        // Simulate a count-down loop:
        //   loop: R[0] = R[0] - R[1];  Lt R[0] R[2] K=0 (R[0] >= R[2] → continue);  Jmp -3
        // Continue while R[0] >= R[2]; side-exit when R[0] < R[2].
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [
            Inst::iabc(Op::Sub, 0, 0, 1, false),
            Inst::iabc(Op::Lt, 0, 2, 0, false), // K=0
            Inst::isj(Op::Jmp, -3),
        ];
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        for (i, inst) in prog.iter().copied().enumerate() {
            rec.push(RecordedOp {
                proto: p,
                pc: i as u32,
                inst,
                inline_depth: 0,
                var_count: None,
            });
        }
        rec.closed = true;
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = 20;
        state[1] = 3; // decrement
        state[2] = 5; // floor

        // Iterate the trace as a real dispatcher would. Expect:
        // 20 → 17, 17 → 14, ..., 8 → 5 (continues; 5 >= 5 still
        // matches K=0 since 5 < 5 is false), 5 → 2 (side-exits
        // because 2 < 5 = true, mismatch).
        let mut iters = 0;
        loop {
            let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
            iters += 1;
            if r != 0 {
                assert_eq!(r, 3, "side-exit PC = cmp_pc(1) + 2");
                break;
            }
            assert!(iters < 100, "loop should terminate");
        }
        assert_eq!(state[0], 2);
        // Iterations: 20→17→14→11→8→5→2 = 6 successful subtracts
        // (5 of them with R[0] >= 5 → continue, then 2 < 5 →
        // side-exit). So iters == 6.
        assert_eq!(iters, 6);
    }
}

#[cfg(test)]
mod s2b_table_ops {
    use super::*;
    use crate::jit_backend::enter_jit;
    use luna_core::version::LuaVersion;
    use luna_core::vm::Vm;
    use luna_core::vm::isa::{Inst, Op};

    const WIDE_SRC: &[u8] = b"local a,b,c,d = 0,0,0,0; return a+b+c+d";

    fn load_proto(vm: &mut Vm, src: &[u8]) -> Gc<Proto> {
        let cl = vm.load(src, b"=t").expect("compile");
        cl.proto
    }

    fn closed_record(proto: Gc<Proto>, head_pc: u32, ops: &[Inst]) -> TraceRecord {
        let mut rec = TraceRecord::start(proto, head_pc, Vec::new(), false);
        for (i, inst) in ops.iter().copied().enumerate() {
            let pushed = rec.push(RecordedOp {
                proto,
                pc: i as u32,
                inst,
                inline_depth: 0,
                var_count: None,
            });
            assert!(pushed);
        }
        rec.closed = true;
        rec
    }

    /// Run a compiled trace under an `enter_jit` guard so the
    /// `luna_jit_*` helpers can reach `vm` via the `JIT_VM`
    /// thread-local.
    ///
    /// SAFETY: `state.len() >= proto.max_stack` is the caller's
    /// invariant — every helper that loads a table-typed slot
    /// dereferences the i64 pointer there, so it must be a real
    /// `Gc<Table>::as_ptr()` (or a `NewTable` op writes one).
    fn run_trace(vm: &mut Vm, ct: &CompiledTrace, state: &mut [i64]) -> i64 {
        let _guard = enter_jit(vm, None);
        unsafe { (ct.entry)(state.as_mut_ptr()) }
    }

    #[test]
    fn new_table_writes_non_null_table_ptr_into_dst() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // Trace: R[0] = {}. The B/C size hints don't matter — the
        // step-4 lowerer reaches for the unsized helper.
        let rec = closed_record(p, 0, &[Inst::iabc(Op::NewTable, 0, 0, 0, false)]);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        let r = run_trace(&mut vm, &ct, &mut state);
        assert_eq!(r, 0);
        assert!(
            state[0] != 0,
            "NewTable must return a non-null Gc<Table> ptr"
        );
        assert!(vm.jit.pending_err.is_none(), "no deopt expected");
    }

    #[test]
    fn set_i_then_get_i_roundtrips_through_a_fresh_table() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // R[0] = {}
        // R[0][1] = R[2]      (SetI with B=1 immediate key)
        // R[3] = R[0][1]       (GetI with C=1 immediate key)
        let rec = closed_record(
            p,
            0,
            &[
                Inst::iabc(Op::NewTable, 0, 0, 0, false),
                Inst::iabc(Op::SetI, 0, 1, 2, false),
                Inst::iabc(Op::GetI, 3, 0, 1, false),
            ],
        );
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[2] = 42; // value to write
        let r = run_trace(&mut vm, &ct, &mut state);
        assert_eq!(r, 0);
        assert!(vm.jit.pending_err.is_none(), "no metatable → no deopt");
        assert_eq!(state[3], 42, "Get must see the value Set wrote");
    }

    #[test]
    fn len_reports_array_size_after_set_i_sequence() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // R[0] = {}
        // R[0][1] = R[1]
        // R[0][2] = R[1]
        // R[0][3] = R[1]
        // R[2] = #R[0]
        let rec = closed_record(
            p,
            0,
            &[
                Inst::iabc(Op::NewTable, 0, 0, 0, false),
                Inst::iabc(Op::SetI, 0, 1, 1, false),
                Inst::iabc(Op::SetI, 0, 2, 1, false),
                Inst::iabc(Op::SetI, 0, 3, 1, false),
                Inst::iabc(Op::Len, 2, 0, 0, false),
            ],
        );
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[1] = 99;
        let r = run_trace(&mut vm, &ct, &mut state);
        assert_eq!(r, 0);
        assert_eq!(state[2], 3, "Len must report array length 3");
    }

    /// A table with a metatable triggers the helper's
    /// `jit_pending_err` short-circuit (PUC routes the write through
    /// `__newindex`; the helper bypasses it, so the lowerer must
    /// deopt instead).
    #[test]
    fn metatable_on_set_i_parks_pending_err() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // Pre-allocate the target table with a metatable.
        let t = vm.heap.new_table();
        let mt = vm.heap.new_table();
        unsafe { t.as_mut() }.set_metatable(Some(mt));

        // Trace: R[0][1] = R[1]. R[0] holds the pre-built table.
        let rec = closed_record(p, 0, &[Inst::iabc(Op::SetI, 0, 1, 1, false)]);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = t.as_ptr() as i64;
        state[1] = 7;
        let r = run_trace(&mut vm, &ct, &mut state);

        assert_eq!(r, 0, "trace still returns head_pc");
        assert!(
            vm.jit.pending_err.is_some(),
            "metatable-bearing table must park a deopt request"
        );
    }

    #[test]
    fn metatable_on_get_i_parks_pending_err() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let t = vm.heap.new_table();
        let mt = vm.heap.new_table();
        unsafe { t.as_mut() }.set_metatable(Some(mt));

        // Trace: R[1] = R[0][1].
        let rec = closed_record(p, 0, &[Inst::iabc(Op::GetI, 1, 0, 1, false)]);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = t.as_ptr() as i64;
        run_trace(&mut vm, &ct, &mut state);

        assert!(vm.jit.pending_err.is_some(), "GetI deopt on metatable");
    }

    #[test]
    fn metatable_on_len_parks_pending_err() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let t = vm.heap.new_table();
        let mt = vm.heap.new_table();
        unsafe { t.as_mut() }.set_metatable(Some(mt));

        let rec = closed_record(p, 0, &[Inst::iabc(Op::Len, 1, 0, 0, false)]);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = t.as_ptr() as i64;
        run_trace(&mut vm, &ct, &mut state);

        assert!(vm.jit.pending_err.is_some(), "Len deopt on metatable");
    }

    /// Once a helper has parked `jit_pending_err`, subsequent
    /// helpers in the same trace early-return without touching
    /// the heap — so the trace can complete safely even though
    /// every subsequent table-op is bogus.
    #[test]
    fn pending_err_short_circuits_downstream_helpers() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let t = vm.heap.new_table();
        let mt = vm.heap.new_table();
        unsafe { t.as_mut() }.set_metatable(Some(mt));

        // SetI (parks err) → SetI (short-circuit) → GetI
        // (short-circuit, returns 0 sentinel).
        let rec = closed_record(
            p,
            0,
            &[
                Inst::iabc(Op::SetI, 0, 1, 1, false),
                Inst::iabc(Op::SetI, 0, 2, 1, false),
                Inst::iabc(Op::GetI, 2, 0, 1, false),
            ],
        );
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = t.as_ptr() as i64;
        state[1] = 11;
        let r = run_trace(&mut vm, &ct, &mut state);

        assert_eq!(r, 0);
        assert!(vm.jit.pending_err.is_some());
        // GetI short-circuit returns 0 sentinel; trace tail stores
        // that back into R[2].
        assert_eq!(state[2], 0);
    }

    #[test]
    fn new_table_dst_out_of_bounds_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // Reg index 200 is way past the proto's max_stack (~5).
        let rec = closed_record(p, 0, &[Inst::iabc(Op::NewTable, 200, 0, 0, false)]);
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn get_i_table_reg_out_of_bounds_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let rec = closed_record(p, 0, &[Inst::iabc(Op::GetI, 0, 200, 1, false)]);
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn set_i_value_reg_out_of_bounds_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let rec = closed_record(p, 0, &[Inst::iabc(Op::SetI, 0, 1, 200, false)]);
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn len_table_reg_out_of_bounds_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let rec = closed_record(p, 0, &[Inst::iabc(Op::Len, 0, 200, 0, false)]);
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }
}

#[cfg(test)]
mod s2b_call_truncation {
    use super::*;
    use luna_core::version::LuaVersion;
    use luna_core::vm::Vm;
    use luna_core::vm::isa::{Inst, Op};

    const WIDE_SRC: &[u8] = b"local a,b,c,d = 0,0,0,0; return a+b+c+d";

    fn load_proto(vm: &mut Vm, src: &[u8]) -> Gc<Proto> {
        let cl = vm.load(src, b"=t").expect("compile");
        cl.proto
    }

    fn closed_record(proto: Gc<Proto>, head_pc: u32, ops: &[Inst]) -> TraceRecord {
        let mut rec = TraceRecord::start(proto, head_pc, Vec::new(), false);
        for (i, inst) in ops.iter().copied().enumerate() {
            let pushed = rec.push(RecordedOp {
                proto,
                pc: i as u32,
                inst,
                inline_depth: 0,
                var_count: None,
            });
            assert!(pushed);
        }
        rec.closed = true;
        rec
    }

    #[test]
    fn single_call_op_side_exits_at_call_pc() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // A trace consisting of one Call at pc=0. Step-5 treats it
        // as a side-exit (no IR is emitted for the Call body).
        let rec = closed_record(
            p,
            5,
            &[Inst::iabc(Op::Call, 0, 1, 1, false)], // call R[0], 0 args, 0 results
        );
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec)
            .expect("Op::Call-only trace must compile");

        let mut state: Vec<i64> = vec![42, 43, 44, 0, 0];
        state.resize(p.max_stack as usize, 0);
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        // Trace's "head_pc" was 5 — but the side-exit at the Call
        // returns the Call's PC (= 0 in this 1-op trace).
        assert_eq!(r, 0, "side-exit at call's PC, not head_pc");
        // Reg state passes through (we loaded + stored every reg).
        assert_eq!(state[0], 42);
        assert_eq!(state[1], 43);
    }

    #[test]
    fn arith_then_call_stores_back_post_arith_state() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // R[0] = R[0] + R[1]; call R[2]
        let prog = [
            Inst::iabc(Op::Add, 0, 0, 1, false),
            Inst::iabc(Op::Call, 2, 1, 0, false),
        ];
        let rec = closed_record(p, 0, &prog);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = 100;
        state[1] = 7;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        // Call is at index 1 → pc 1.
        assert_eq!(r, 1);
        // The Add ran before the Call's side-exit, so the
        // post-Add value lives in reg_state[0].
        assert_eq!(state[0], 107, "post-arith state visible to interp");
        assert_eq!(state[1], 7);
    }

    #[test]
    fn ops_after_first_call_are_silently_dropped() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // [Add, Call, Mul] — Mul never executes; specifically, a
        // Mul reading R[200] (out of bounds) would normally bail
        // the lowerer, but step-5 skips post-truncation ops so it
        // compiles fine.
        let prog = [
            Inst::iabc(Op::Add, 0, 1, 2, false),
            Inst::iabc(Op::Call, 3, 1, 0, false),
            Inst::iabc(Op::Mul, 0, 200, 200, false), // would-be OOB if validated
        ];
        let rec = closed_record(p, 0, &prog);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec)
            .expect("compile despite post-truncation OOB");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[1] = 30;
        state[2] = 12;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        assert_eq!(r, 1, "side-exit at Call.pc = 1");
        // The Mul never ran — R[0] holds the Add's result, not
        // R[0]*200.
        assert_eq!(state[0], 42);
    }

    #[test]
    fn cmp_immediately_before_call_bails() {
        // The cmp's "took the Jmp" recording requires a Jmp at
        // cmp_pc + 1. If that slot is an Op::Call instead, the
        // recorded direction can't be lowered as step-5 understands
        // it — bail.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [
            Inst::iabc(Op::Lt, 0, 1, 0, true),
            Inst::iabc(Op::Call, 2, 1, 0, false),
        ];
        let rec = closed_record(p, 0, &prog);
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn call_a_register_out_of_bounds_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // p.max_stack ≈ 5; A=200 is way past it.
        let rec = closed_record(p, 0, &[Inst::iabc(Op::Call, 200, 1, 0, false)]);
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn cmp_then_jmp_then_call_truncation() {
        // [Lt, Jmp, Add, Call] — the cmp + Jmp pair behaves
        // normally (Jmp consumed by cmp); Add runs; Call truncates.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [
            Inst::iabc(Op::Lt, 0, 1, 0, true),
            Inst::isj(Op::Jmp, -1),
            Inst::iabc(Op::Add, 0, 0, 2, false),
            Inst::iabc(Op::Call, 3, 1, 0, false),
        ];
        let rec = closed_record(p, 0, &prog);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        // R[0]=5, R[1]=10 → 5 < 10 holds → cmp matches K=1 → continue.
        // R[0] += R[2] = 5 + 7 = 12.
        // Call at pc 3 side-exits.
        state[0] = 5;
        state[1] = 10;
        state[2] = 7;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        assert_eq!(r, 3, "side-exit at Call.pc = 3");
        assert_eq!(state[0], 12);

        // Now flip so cmp doesn't match: R[0]=99, R[1]=10 → 99<10
        // false → side-exit at cmp_pc+2 = 2 (NOT the Call's pc).
        state[0] = 99;
        state[1] = 10;
        state[2] = 7;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        assert_eq!(r, 2, "cmp side-exit takes precedence over Call truncation");
        assert_eq!(state[0], 99, "Add never ran on this path");
    }

    #[test]
    fn forloop_continues_internal_loop_until_count_hits_zero() {
        // Two-block trace shape: a body op (Add) plus the trailing
        // ForLoop. Internal loop runs natively until count == 0,
        // then side-exits at forloop.pc + 1.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [
            Inst::iabc(Op::Add, 0, 0, 3, false),     // R[0] += R[3]
            Inst::iabc(Op::ForLoop, 1, 0, 0, false), // ForLoop on R[1..R[1+3]]
        ];
        let rec = closed_record(p, 0, &prog);
        let opts = CompileOptions {
            internal_loop: true,
            pre53: false,
            aot: false,
        };
        let ct =
            try_compile_trace_with_options(vm.jit.storage.as_mut(), &rec, opts).expect("compile");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        // R[0] = accumulator, R[3] = step for the body's add.
        // R[1] = cur loop var (init 1), R[2] = count (5 iters),
        // R[3] = for-loop step (1), R[4] = visible loop var.
        // BUT R[3] is shared between the body's Add and ForLoop's
        // step — both are 1, so the test setup happens to work
        // (a realistic Proto would use disjoint slots).
        state[0] = 0;
        state[1] = 1;
        state[2] = 5; // count
        state[3] = 1; // step / body add operand
        state[4] = 0;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        // ForLoop at index 1, pc=1; exit PC = pc+1 = 2.
        assert_eq!(r, 2, "ForLoop count-exhaustion side-exits at pc+1");
        // Body order: Add runs BEFORE ForLoop's count check. With
        // count = 5 initially, body iters: count=5 (Add → 1), 4, 3,
        // 2, 1, 0 (ForLoop's check sees 0, side-exits AFTER the
        // Add ran). So Add fires 6 times — R[0] = 6.
        assert_eq!(state[0], 6);
        // R[2] (count) decremented to 0.
        assert_eq!(state[2], 0);
    }

    #[test]
    fn forloop_one_shot_returns_body_pc_on_continue() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // ForLoop only — empty body, single iter dispatch.
        let prog = [Inst::iabc(Op::ForLoop, 0, 0, 0, false)];
        let rec = closed_record(p, 9, &prog);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec)
            .expect("compile one-shot ForLoop trace");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        state[0] = 10;
        state[1] = 3; // count > 0 → continue
        state[2] = 1; // step
        state[3] = 0;
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        // Per the nested-loop bug fix (docs/known-bugs/trace-jit-
        // nested-loop-wrong-result.md), ForLoop continue returns the
        // BODY START pc = (rop.pc + 1) - bx, not head_pc. In this
        // synthetic test the closed_record helper assigns sequential
        // pcs starting from 0, so rop.pc=0, bx=0, body_pc=1. The
        // earlier test expected head_pc=9 which was the buggy
        // behavior — re-dispatching the same ForLoop op would double-
        // advance the counter (the nested-loop bug fix). For a real
        // loop with non-zero bx, body_pc would land on the loop body
        // start; for this synthetic op chain body_pc just exits past
        // the ForLoop.
        assert_eq!(r, 1, "one-shot continue returns body_pc=(rop.pc+1)-bx=1");
        // R[0] = 10 + 1 = 11.
        assert_eq!(state[0], 11);
        assert_eq!(state[1], 2); // count -= 1
        assert_eq!(state[3], 11); // visible loop var = next
    }

    #[test]
    fn forloop_pre53_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [Inst::iabc(Op::ForLoop, 0, 0, 0, false)];
        let rec = closed_record(p, 0, &prog);
        let opts = CompileOptions {
            internal_loop: false,
            pre53: true,
            aot: false,
        };
        assert!(try_compile_trace_with_options(vm.jit.storage.as_mut(), &rec, opts).is_none());
    }

    #[test]
    fn forloop_a_plus_3_out_of_bounds_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // max_stack = 5; A=3 → A+3 = 6 ≥ 5 → bail.
        let prog = [Inst::iabc(Op::ForLoop, 3, 0, 0, false)];
        let rec = closed_record(p, 0, &prog);
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn op_call_then_unwhitelisted_op_still_compiles() {
        // Even if a non-whitelisted op (Op::Return0) appears after
        // the first Call, the truncation drops it before validation.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let prog = [
            Inst::iabc(Op::Call, 0, 1, 0, false),
            Inst::iabc(Op::Return0, 0, 0, 0, false), // would bail if validated
        ];
        let rec = closed_record(p, 0, &prog);
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec)
            .expect("compile despite post-truncation Return0");

        let mut state: Vec<i64> = vec![0; p.max_stack as usize];
        let r = unsafe { (ct.entry)(state.as_mut_ptr()) };
        assert_eq!(r, 0);
    }
}

#[cfg(test)]
mod s4_step3a_op_offsets {
    //! P12-S4-step3a — `compute_op_offsets` correctness tests.
    //!
    //! The helper is a pure function over `TraceRecord.ops`'s
    //! `inline_depth` field. These tests build synthetic records
    //! with hand-set depths + `Op::Call` ops to verify the offset
    //! stack matches the expected per-frame register window base.
    //! Step 3b will start consuming the helper's output in the
    //! body emit pass.

    use super::*;
    use luna_core::version::LuaVersion;
    use luna_core::vm::Vm;
    use luna_core::vm::isa::{Inst, Op};

    fn load_proto(vm: &mut Vm, src: &[u8]) -> Gc<Proto> {
        vm.load(src, b"=t").expect("compile").proto
    }

    fn make_record(proto: Gc<Proto>, items: Vec<(Inst, u8)>) -> TraceRecord {
        let mut rec = TraceRecord::start(proto, 0, Vec::new(), true);
        for (i, (inst, depth)) in items.into_iter().enumerate() {
            let pushed = rec.push(RecordedOp {
                proto,
                pc: i as u32,
                inst,
                inline_depth: depth,
                var_count: None,
            });
            assert!(pushed, "rec.push should not overflow");
        }
        rec
    }

    /// Pure depth-0 trace → all offsets stay at 0.
    #[test]
    fn depth_zero_only_yields_zero_offsets() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, b"return 0");
        let rec = make_record(
            p,
            vec![
                (Inst::iabc(Op::LoadI, 0, 0, 128, false), 0),
                (Inst::iabc(Op::Add, 1, 0, 0, false), 0),
                (Inst::iabc(Op::Return1, 1, 0, 0, false), 0),
            ],
        );
        let (offsets, enclosing) = compute_op_offsets(&rec);
        assert_eq!(offsets, vec![0u32, 0, 0]);
        assert_eq!(enclosing, vec![None, None, None]);
    }

    /// Depth 0 → 1 → 0 via Op::Call(A=3) then Op::Return.
    /// Callee's offset = 0 + 3 + 1 = 4; back to depth 0 → 0.
    #[test]
    fn single_call_bumps_then_drops_offset() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, b"return 0");
        let rec = make_record(
            p,
            vec![
                (Inst::iabc(Op::Move, 0, 0, 0, false), 0),
                (Inst::iabc(Op::Call, 3, 2, 2, false), 0), // A=3
                (Inst::iabc(Op::LoadI, 0, 0, 128, false), 1), // callee R[0]
                (Inst::iabc(Op::Return1, 0, 0, 0, false), 1),
                (Inst::iabc(Op::Move, 4, 3, 0, false), 0), // back to depth 0
            ],
        );
        let (offsets, enclosing) = compute_op_offsets(&rec);
        assert_eq!(offsets, vec![0u32, 0, 4, 4, 0]);
        // depth 0,0,1,1,0 → enclosing 0,0,Some(3),Some(3),0 (A=3 from Op::Call)
        assert_eq!(enclosing, vec![None, None, Some(3), Some(3), None]);
    }

    /// Nested calls: depth 0 → 1 → 2. Each Op::Call A is captured.
    /// Caller's A=2 → callee_1 offset = 0+2+1 = 3.
    /// Callee_1's A=4 → callee_2 offset = 3+4+1 = 8.
    #[test]
    fn nested_calls_accumulate_offsets() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, b"return 0");
        let rec = make_record(
            p,
            vec![
                (Inst::iabc(Op::Call, 2, 1, 0, false), 0),
                (Inst::iabc(Op::Call, 4, 1, 0, false), 1),
                (Inst::iabc(Op::LoadI, 0, 0, 128, false), 2),
                (Inst::iabc(Op::Return0, 0, 0, 0, false), 2),
                (Inst::iabc(Op::Return0, 0, 0, 0, false), 1),
                (Inst::iabc(Op::Move, 0, 0, 0, false), 0),
            ],
        );
        let (offsets, enclosing) = compute_op_offsets(&rec);
        // Expected:
        //   ops[0] = Call A=2,           depth=0 → offset 0
        //   ops[1] = Call A=4 (callee_1) depth=1 → offset = 0+2+1 = 3
        //   ops[2] = LoadI (callee_2)    depth=2 → offset = 3+4+1 = 8
        //   ops[3] = Return0             depth=2 → offset 8
        //   ops[4] = Return0             depth=1 → offset 3
        //   ops[5] = Move                depth=0 → offset 0
        assert_eq!(offsets, vec![0u32, 3, 8, 8, 3, 0]);
        assert_eq!(
            enclosing,
            vec![None, Some(2), Some(4), Some(4), Some(2), None]
        );
    }
}

#[cfg(test)]
mod s4_step3b_inline_emit {
    //! P12-S4-step3b — body emit consumes the offset/enclosing/window
    //! triple from `compute_op_offsets`. These tests craft synthetic
    //! `TraceRecord`s with depth>0 ops (the recorder doesn't produce
    //! them on real Lua code yet — step 4's job) to verify the new
    //! emit paths in isolation.
    use super::*;
    use luna_core::version::LuaVersion;
    use luna_core::vm::Vm;
    use luna_core::vm::isa::{Inst, Op};

    const WIDE_SRC: &[u8] = b"local a,b,c,d = 0,0,0,0; return a+b+c+d";

    fn load_proto(vm: &mut Vm, src: &[u8]) -> Gc<Proto> {
        vm.load(src, b"=t").expect("compile").proto
    }

    /// step4b-C-2 supersedes the InlineAbort-for-cmp behavior: a
    /// cmp@d>0 now emits a real side-exit via the frame-mat helper.
    /// The trace is dispatchable (subject to other gates). See
    /// `s4_step4b_skeleton::per_exit_metas_populated_for_cmp_at_depth_one`
    /// for the positive coverage; this slot is kept as a regression
    /// guard against accidentally re-introducing the InlineAbort
    /// path for cmp@d>0.
    #[test]
    fn cmp_at_depth_one_no_longer_aborts_via_inline_abort() {
        // Trace with a non-vararg head proto containing one
        // self-rec Call followed by a cmp+Jmp at depth=1. With the
        // step4b-C-2 emit path, end_idx_opt finds NO InlineAbort
        // terminator at the cmp — it falls through to normal emit.
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        // Find a non-vararg inner proto: `local function f(a,b)
        // return a+b end` keeps the inner Proto non-vararg.
        let cl = vm
            .load(b"local function f(a,b) return a+b end return f", b"=t")
            .expect("compile");
        let p = cl.proto.protos[0];
        assert!(!p.is_vararg, "fixture must be non-vararg");
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Add, 0, 1, 2, false),
            inline_depth: 0,
            var_count: None,
        });
        rec.push(RecordedOp {
            proto: p,
            pc: 1,
            inst: Inst::iabc(Op::Call, 0, 1, 2, false),
            inline_depth: 0,
            var_count: None,
        });
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Lt, 0, 1, 0, true),
            inline_depth: 1,
            var_count: None,
        });
        rec.push(RecordedOp {
            proto: p,
            pc: 1,
            inst: Inst::isj(Op::Jmp, 0),
            inline_depth: 1,
            var_count: None,
        });
        rec.closed = true;
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec)
            .expect("cmp@d>0 now compiles via inline-cmp emit");
        // One cmp@d>0 site → one per-exit-metas entry.
        assert_eq!(ct.per_exit_inline.len(), 1);
        // Window covers caller + inlined frame's slots.
        assert!(
            ct.window_size as usize >= p.max_stack as usize,
            "window_size ({}) must be ≥ max_stack ({})",
            ct.window_size,
            p.max_stack,
        );
    }

    /// First op at depth>0 violates the recorder invariant and must
    /// bail cleanly — `compute_op_offsets` would otherwise underflow
    /// the depth-bump arithmetic.
    #[test]
    fn first_op_at_depth_gt_zero_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Add, 0, 1, 2, false),
            inline_depth: 1, // invariant violation
            var_count: None,
        });
        rec.closed = true;
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    /// First op on a different proto than the trace head also bails
    /// (cross-proto on the head op is an invalid recorder state).
    #[test]
    fn first_op_on_cross_proto_bails() {
        let mut vm1 = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let mut vm2 = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p1 = load_proto(&mut vm1, WIDE_SRC);
        let p2 = load_proto(&mut vm2, WIDE_SRC);
        let mut rec = TraceRecord::start(p1, 0, Vec::new(), false);
        rec.push(RecordedOp {
            proto: p2,
            pc: 0,
            inst: Inst::iabc(Op::Add, 0, 1, 2, false),
            inline_depth: 0,
            var_count: None,
        });
        rec.closed = true;
        assert!(try_compile_trace(vm1.jit.storage.as_mut(), &rec).is_none());
    }
}

#[cfg(test)]
mod s4_step4b_skeleton {
    //! P12-S4-step4b-A — frame-mat helper + data structures wired
    //! but no IR emit site yet. These tests pin the contracts the
    //! later sub-steps build on:
    //!   - FrameMaterializeInfo layout is repr(C) (12 bytes amd64)
    //!   - CompiledTrace.frame_metas is empty for all current
    //!     production trace shapes (no path through the lowerer
    //!     populates it yet)
    //!   - helper symbol resolves and the skeleton returns -1
    //!     (preventing any accidental call from advancing past
    //!     deopt until step4b-B fills the body)
    use super::*;
    use luna_core::version::LuaVersion;
    use luna_core::vm::Vm;
    use luna_core::vm::isa::{Inst, Op};

    const WIDE_SRC: &[u8] = b"local a,b,c,d = 0,0,0,0; return a+b+c+d";

    fn load_proto(vm: &mut Vm, src: &[u8]) -> Gc<Proto> {
        vm.load(src, b"=t").expect("compile").proto
    }

    #[test]
    fn frame_materialize_info_layout_is_stable() {
        // 12-byte layout on every supported target — 4 + 4 + 4. If
        // padding ever sneaks in here the IR's pointer-arithmetic
        // load in step4b-C would read garbage.
        assert_eq!(std::mem::size_of::<FrameMaterializeInfo>(), 12);
        assert_eq!(std::mem::align_of::<FrameMaterializeInfo>(), 4);
    }

    #[test]
    fn compiled_traces_have_empty_per_exit_metas_when_no_inline_cmp() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        // A plain depth=0 add — no cmp@d>0 site, so per_exit_inline
        // stays empty.
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Add, 0, 1, 2, false),
            inline_depth: 0,
            var_count: None,
        });
        rec.closed = true;
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("simple add compiles");
        assert!(
            ct.per_exit_inline.is_empty(),
            "no cmp@d>0 site → per_exit_inline empty"
        );
    }

    #[test]
    fn helper_with_no_lua_frame_returns_deopt() {
        // No `enter_jit` guard + no Lua frame at trace head → helper
        // hits the `jit_last_lua_frame()` None branch and returns -1
        // (deopt sentinel). We seed an enter_jit so `current_jit_vm`
        // resolves but leave `vm.frames` empty (test-only Vm starts
        // with no Lua frame).
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        // Need a closure pointer for `current_jit_closure`. Load a
        // tiny chunk and pin its main closure.
        let cl = vm.load(WIDE_SRC, b"=t").expect("compile");
        let metas: [FrameMaterializeInfo; 0] = [];
        let r = {
            let _g = crate::jit_backend::enter_jit(&mut vm, Some(cl));
            unsafe { super::super::luna_jit_trace_materialize_frames(0, metas.as_ptr()) }
        };
        assert_eq!(r, -1, "no live Lua frame → helper returns deopt sentinel");
    }

    /// step4b-B: helper with a live trace-head frame pushes N inlined
    /// frames with `base = head.base + meta.base_offset`, `pc` from
    /// the meta, `func_slot = base - 1`, and `nresults` from the
    /// meta. `cl.proto` is the same closure pinned by enter_jit.
    #[test]
    fn helper_pushes_one_inlined_frame_with_correct_metadata() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        // Run a tiny program so vm.frames ends with a live Lua frame
        // — actually, after `eval` returns the frames are popped.
        // Easier route: drive a recording-like fixture. Use the
        // existing `enter_jit_dispatch` machinery by hand isn't
        // worth it for a unit; instead seed a frame directly.
        let cl = vm.load(WIDE_SRC, b"=t").expect("compile");
        // Push a synthetic head frame so `jit_last_lua_frame` returns Some.
        vm.jit_ensure_stack(64);
        vm.jit_push_inlined_frame(cl, /*base*/ 1, /*pc*/ 7, /*nresults*/ 1);
        let frames_before = {
            // borrow vm just to count — use the accessor via guard
            // scope below to compose without lifetime pain.
            let _g = crate::jit_backend::enter_jit(&mut vm, Some(cl));
            // can't call vm methods inside _g scope (vm moved into
            // accessor); drop guard first.
            drop(_g);
            // recompute after guard drop
            // (intentional: count via the Vm public surface — there
            // isn't a frame_count() public accessor, so we use the
            // fact that the helper returns 0 for success and the
            // post-push assertion below covers count via the
            // last-frame's base.)
            0
        };
        let _ = frames_before;
        let metas = [FrameMaterializeInfo {
            base_offset: 5,
            pc: 11,
            nresults: 1,
        }];
        let r = {
            let _g = crate::jit_backend::enter_jit(&mut vm, Some(cl));
            unsafe { super::super::luna_jit_trace_materialize_frames(1, metas.as_ptr()) }
        };
        assert_eq!(r, 0, "successful push returns 0");
        // The just-pushed frame has base = head.base + 5 = 1 + 5 = 6,
        // pc = 11, nresults = 1, func_slot = base - 1 = 5.
        let pushed = vm.jit_last_lua_frame().expect("frame was pushed");
        assert_eq!(pushed.base, 6);
        assert_eq!(pushed.pc, 11);
        assert_eq!(pushed.func_slot, 5);
        assert_eq!(pushed.nresults, 1);
        assert_eq!(pushed.n_varargs, 0);
    }

    /// step4b-C-2: a single self-recursive Call followed by a cmp@d=1
    /// produces ONE per_exit_metas entry — the cmp's chain has one
    /// frame (the inlined callee) with base_offset matching
    /// op_offsets[2] and pc overridden to the cmp's side-exit PC
    /// (`cmp.pc + 2`, the TookJmp direction).
    #[test]
    fn per_exit_metas_populated_for_cmp_at_depth_one() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        // Non-vararg inner proto needed — step4b-C-2's vararg bail
        // refuses the self-rec inline path on a vararg head.
        let cl = vm
            .load(b"local function f(a,b) return a+b end return f", b"=t")
            .expect("compile");
        let p = cl.proto.protos[0];
        assert!(!p.is_vararg);
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        // depth 0: an Add, then a self-recursive Call.
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Add, 0, 1, 2, false),
            inline_depth: 0,
            var_count: None,
        });
        rec.push(RecordedOp {
            proto: p,
            pc: 1,
            inst: Inst::iabc(Op::Call, 0, 1, 2, false), // A=0, C=2 → nresults 1
            inline_depth: 0,
            var_count: None,
        });
        // depth 1: a cmp + the trailing Jmp the cmp consumes
        // (TookJmp direction). Side-exit PC = cmp.pc + 2 = 2.
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Lt, 0, 1, 0, true),
            inline_depth: 1,
            var_count: None,
        });
        rec.push(RecordedOp {
            proto: p,
            pc: 1,
            inst: Inst::isj(Op::Jmp, 0),
            inline_depth: 1,
            var_count: None,
        });
        rec.closed = true;
        let ct = try_compile_trace(vm.jit.storage.as_mut(), &rec).expect("compiles via inline-cmp");
        assert_eq!(
            ct.per_exit_inline.len(),
            1,
            "one cmp@d>0 site → one per-exit-inline entry"
        );
        let info = &ct.per_exit_inline[0];
        // Cmp at depth=1 pc=0; TookJmp → side-exit pc = cmp.pc + 2 = 2.
        assert_eq!(info.cont_pc, 2);
        assert_eq!(info.chain.len(), 1, "one inlined frame in chain");
        let m = info.chain[0];
        // Call A=0 → callee base_offset = A + 1 = 1.
        assert_eq!(m.base_offset, 1);
        // Innermost frame's pc was overridden from caller-resume
        // (= Call.pc + 1 = 2) to the side-exit pc (also 2 here, but
        // could differ for SkippedJmp or non-trivial layouts).
        assert_eq!(m.pc, 2);
        assert_eq!(m.nresults, 1);
    }

    /// step4b-C-1: Op::Call with C != 2 (i.e. nresults != 1) bails
    /// the whole trace — step3b's Op::Return1 copy-back assumes one
    /// value, and the helper passes through whatever `nresults` the
    /// meta says without validating.
    #[test]
    fn self_recursive_call_with_multiple_returns_bails() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let p = load_proto(&mut vm, WIDE_SRC);
        let mut rec = TraceRecord::start(p, 0, Vec::new(), false);
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Call, 0, 1, 3, false), // C=3 → nresults=2
            inline_depth: 0,
            var_count: None,
        });
        rec.push(RecordedOp {
            proto: p,
            pc: 0,
            inst: Inst::iabc(Op::Add, 0, 1, 2, false),
            inline_depth: 1,
            var_count: None,
        });
        rec.closed = true;
        assert!(try_compile_trace(vm.jit.storage.as_mut(), &rec).is_none());
    }

    #[test]
    fn helper_pushes_multiple_frames_in_order() {
        let mut vm = crate::jit_backend::test_vm_new(LuaVersion::Lua55);
        let cl = vm.load(WIDE_SRC, b"=t").expect("compile");
        vm.jit_ensure_stack(64);
        vm.jit_push_inlined_frame(cl, 1, 0, 1);
        let metas = [
            FrameMaterializeInfo {
                base_offset: 3,
                pc: 7,
                nresults: 1,
            },
            FrameMaterializeInfo {
                base_offset: 8,
                pc: 7,
                nresults: 1,
            },
            FrameMaterializeInfo {
                base_offset: 13,
                pc: 9,
                nresults: 1,
            },
        ];
        let r = {
            let _g = crate::jit_backend::enter_jit(&mut vm, Some(cl));
            unsafe { super::super::luna_jit_trace_materialize_frames(3, metas.as_ptr()) }
        };
        assert_eq!(r, 0);
        // Innermost frame should match metas[2].
        let inner = vm.jit_last_lua_frame().expect("inner frame");
        assert_eq!(inner.base, 1 + 13);
        assert_eq!(inner.pc, 9);
    }
}

#[cfg(test)]
mod s6_step_a1 {
    //! P12-S6-A1 — `ExitTag::Nil` variant exists and
    //! `kinds_to_exit_tags` produces it for `RegKind::Nil`. Foundation
    //! for the S6-A2 LoadNil emit (which actually writes Nil to a
    //! slot whose entry tag may not be Nil).
    use super::*;

    #[test]
    fn regkind_nil_maps_to_exittag_nil() {
        let kinds = vec![
            RegKind::Unset,
            RegKind::Int,
            RegKind::Nil,
            RegKind::Float,
            RegKind::Nil,
        ];
        let tags = kinds_to_exit_tags(&kinds);
        assert_eq!(tags.len(), 5);
        assert!(matches!(tags[0], ExitTag::Untouched));
        assert!(matches!(tags[1], ExitTag::Int));
        assert!(matches!(tags[2], ExitTag::Nil));
        assert!(matches!(tags[3], ExitTag::Float));
        assert!(matches!(tags[4], ExitTag::Nil));
    }
}