vj-codegen 0.1.0

//! The codegen side of VPI's dynamic objects and callbacks (IEEE 1800-2023
//! 37.43 frames, 37.44 threads, 38.36.1 `cbStmt`).
//!
//! Everything here is inert unless the embedder calls [`set_enabled`]`(true)`
//! before lowering (`run_design` does so only when a VPI library is loaded):
//! a design run without one gets no probe, no table and no extra symbol, so
//! it neither slows down nor changes.
//!
//! A statement is identified across the two sides by its [`StmtKey`] -- its
//! source range and kind, which the VPI model records for each statement
//! object -- and an LLVM probe carries only a *site number* into the tables
//! this module accumulates while the design is lowered.

use std::cell::{Cell, RefCell};
use std::collections::HashMap;

pub use sv_sim_exec::vpi_data::{
    FramePart, FrameRef, FrameSite, FrameVar, OP_DISABLE, OP_FRAME_ENTER, OP_FRAME_EXIT,
    OP_FRAME_REBIND, OP_REF_BIND, OP_STMT, StmtKey, VarClass,
};

use crate::types::TypeWidth;
use sv_lang::Statement;
use sv_lang::kinds::StatementKind;
use sv_sim_sim::symid::SymbolUid;

/// The statements `st` directly contains, with lists flattened.
fn flat_children<'d>(st: &Statement<'d>, out: &mut Vec<Statement<'d>>) {
    for c in sv_sim_sim::process::cfg::live_statements(st) {
        if c.kind() == StatementKind::List {
            flat_children(&c, out);
        } else {
            out.push(c);
        }
    }
}

/// Blocks nested directly inside block `st` with the same source range.
fn nest_of(st: &Statement<'_>) -> u32 {
    if st.kind() != StatementKind::Block {
        return 0;
    }
    let mut kids = Vec::new();
    flat_children(st, &mut kids);
    let (r, k) = (st.source_range(), kids.as_slice());
    match k {
        [one] if one.kind() == StatementKind::Block => {
            let q = one.source_range();
            if (q.start.buffer, q.start.offset, q.end.offset)
                == (r.start.buffer, r.start.offset, r.end.offset)
            {
                1 + nest_of(one)
            } else {
                0
            }
        }
        _ => 0,
    }
}

/// The key of `st`.
pub fn key_of(st: &Statement<'_>) -> StmtKey {
    let r = st.source_range();
    StmtKey {
        buffer: r.start.buffer,
        start: r.start.offset,
        end: r.end.offset,
        kind: st.kind() as u16,
        nest: nest_of(st),
    }
}

/// The runtime symbol every probe calls: `void sv_rt_vpi_dyn(u32 op, u32 a,
/// u64 b)`. `op` selects the probe (`OP_*`), `a`/`b` are its operands.
pub const SV_RT_VPI_DYN_SYMBOL: &str = "sv_rt_vpi_dyn";

thread_local! {
    static ENABLED: Cell<bool> = const { Cell::new(false) };
    /// Fork branch kernel index -> the branch statement it runs.
    static FORK_ORIGINS: RefCell<HashMap<u32, StmtKey>> = RefCell::new(HashMap::new());
    /// Statement site number -> the statement, and the reverse.
    static STMT_SITES: RefCell<(Vec<StmtKey>, HashMap<StmtKey, u32>)> =
        RefCell::new((Vec::new(), HashMap::new()));
    /// `disable` site number -> the `disable` statement and the label it names.
    static DISABLE_SITES: RefCell<Vec<(StmtKey, String)>> = const { RefCell::new(Vec::new()) };
    /// Frame site number -> the subroutine and its automatic variables.
    static FRAME_SITES: RefCell<Vec<FrameSite>> = const { RefCell::new(Vec::new()) };
    /// Each frame variable's number ([`FrameVar::id`]).
    static VAR_IDS: RefCell<HashMap<sv_sim_sim::symid::SymbolId, u32>> =
        RefCell::new(HashMap::new());
    /// Set by the process lowering just before it lowers a statement the CFG
    /// already probed ([`mark_probed_by_cfg`]); the statement lowering takes
    /// it on entry ([`emit_entry_probe`]).
    static PROBED_BY_CFG: Cell<bool> = const { Cell::new(false) };
}

/// Turns the probes on or off for the design about to be lowered, and
/// forgets the tables of any earlier one.
pub fn set_enabled(on: bool) {
    ENABLED.with(|e| e.set(on));
    FORK_ORIGINS.with(|f| f.borrow_mut().clear());
    STMT_SITES.with(|s| *s.borrow_mut() = (Vec::new(), HashMap::new()));
    DISABLE_SITES.with(|s| s.borrow_mut().clear());
    FRAME_SITES.with(|s| s.borrow_mut().clear());
    VAR_IDS.with(|s| s.borrow_mut().clear());
    PROBED_BY_CFG.with(|p| p.set(false));
}

/// Whether probes are being emitted.
#[inline]
pub fn enabled() -> bool {
    ENABLED.with(Cell::get)
}

/// Records that fork branch kernel `kernel` runs the statement `branch`.
pub fn note_fork_branch(kernel: u32, branch: &Statement<'_>) {
    if enabled() {
        let key = key_of(branch);
        FORK_ORIGINS.with(|f| f.borrow_mut().insert(kernel, key));
    }
}

/// The fork branches lowered since [`set_enabled`], by kernel index.
pub fn take_fork_origins() -> HashMap<u32, StmtKey> {
    FORK_ORIGINS.with(|f| std::mem::take(&mut *f.borrow_mut()))
}

/// The statements probed since [`set_enabled`], indexed by site number.
pub fn take_stmt_sites() -> Vec<StmtKey> {
    STMT_SITES.with(|s| std::mem::take(&mut s.borrow_mut().0))
}

/// The `disable` statements probed since [`set_enabled`] with the label each
/// names, indexed by site number.
pub fn take_disable_sites() -> Vec<(StmtKey, String)> {
    DISABLE_SITES.with(|s| std::mem::take(&mut *s.borrow_mut()))
}

/// The frame sites probed since [`set_enabled`], indexed by site number.
pub fn take_frame_sites() -> Vec<FrameSite> {
    FRAME_SITES.with(|s| std::mem::take(&mut *s.borrow_mut()))
}

/// The VPI class of a variable of type `ty` (mirrors the VPI model's own
/// classification of variables).
fn var_class_of(ty: &sv_lang::Type<'_>) -> VarClass {
    use sv_lang::{FloatKind, PredefinedIntegerKind as P};
    let ty = ty.canonical();
    if ty.is_event() {
        VarClass::Event
    } else if ty.is_class() {
        VarClass::Class
    } else if ty.is_chandle() {
        VarClass::Chandle
    } else if ty.is_floating() {
        match ty.floating_kind() {
            FloatKind::ShortReal => VarClass::ShortReal,
            FloatKind::Real | FloatKind::RealTime => VarClass::Real,
        }
    } else if ty.is_string() {
        VarClass::String
    } else if ty.is_unpacked_array() {
        if ty.element_type().is_some_and(|e| e.canonical().is_event()) {
            VarClass::EventArray
        } else {
            VarClass::Array
        }
    } else if ty.is_enum() {
        VarClass::Enum
    } else if ty.is_packed_union() || ty.is_unpacked_union() {
        VarClass::Union
    } else if ty.is_struct() {
        VarClass::Struct
    } else if ty.is_packed_array()
        && ty.element_type().is_some_and(|e| {
            let e = e.canonical();
            e.is_struct() || e.is_packed_union() || e.is_enum() || e.is_packed_array()
        })
    {
        VarClass::PackedArray
    } else if ty.is_predefined_integer() {
        match ty.predefined_integer_kind() {
            P::ShortInt => VarClass::ShortInt,
            P::Int => VarClass::Int,
            P::LongInt => VarClass::LongInt,
            P::Byte => VarClass::Byte,
            P::Integer => VarClass::Integer,
            P::Time => VarClass::Time,
        }
    } else if ty.is_four_state() {
        VarClass::Logic
    } else {
        VarClass::Bit
    }
}

/// The declared packed range `(left, right)` of an integral type (`logic
/// [8:1]` is `(8, 1)`, a plain integer type `(w - 1, 0)`); `None` for any
/// other type. Names a bit or part select of it (37.17).
fn declared_range(t: &sv_lang::Type<'_>) -> Option<(i32, i32)> {
    let c = t.canonical();
    if c.is_packed_array() {
        let r = c.fixed_range();
        Some((r.left, r.right))
    } else if c.is_integral() && !c.is_unpacked_array() {
        Some((c.bit_width() as i32 - 1, 0))
    } else {
        None
    }
}

/// The byte distance between two elements of an LLVM array of `bits`-wide
/// integers (the ABI size of `iN` on the 64-bit hosts the JIT and the AOT
/// target: a power of two up to 8 bytes, then a multiple of 16).
fn int_stride(bits: u32) -> u32 {
    let bytes = bits.max(1).div_ceil(8);
    if bytes <= 8 {
        bytes.next_power_of_two()
    } else {
        bytes.next_multiple_of(16)
    }
}

/// Appends the field parts of the unpacked struct or (untagged) union `ty`,
/// whose stored integer is `cwidth` bits wide at byte `byte` and whose own
/// lowest bit within it is `bit0`, below part `parent`; a nested struct or
/// union field is a part of its own with its fields below it. A field is an
/// integral or real scalar or a string; `false` for any other field type
/// (nothing useful is then appended).
fn push_fields(
    ty: &sv_lang::Type<'_>,
    (byte, cwidth, bit0): (u32, u32, u32),
    parent: Option<u32>,
    out: &mut Vec<FramePart>,
) -> bool {
    use sv_sim_sim::state::storage;
    for f in ty.fields() {
        let (Some(fty), Some((lo, _))) =
            (f.value_type(), storage::field_slice(ty, f.field_index()))
        else {
            return false;
        };
        let fty = fty.canonical();
        let bit = bit0 + lo as u32;
        let nested = fty.is_unpacked_struct() || fty.is_unpacked_union();
        if nested && fty.is_tagged_union() {
            return false;
        }
        if !nested
            && !((fty.is_integral() || fty.is_floating() || fty.is_string())
                && !fty.is_unpacked_array())
        {
            return false;
        }
        let class = var_class_of(&fty);
        let width = match class {
            VarClass::Real | VarClass::String => 64,
            VarClass::ShortReal => 32,
            _ if nested => storage::storage_width(&fty).map_or(0, |w| w as u32),
            _ => fty.value_width(),
        };
        out.push(FramePart {
            name: f.name().to_owned(),
            class,
            width,
            signed: fty.is_signed(),
            offset: bit,
            index: 0,
            parent,
            byte,
            cwidth,
            bit,
            range: declared_range(&fty),
        });
        if nested {
            let me = (out.len() - 1) as u32;
            if !push_fields(&fty, (byte, cwidth, bit), Some(me), out) {
                return false;
            }
        }
    }
    true
}

/// The fields of an unpacked struct or untagged union (integral, real and
/// string fields, nested structs and unions) or the rows of a fixed unpacked
/// array of integral, real, struct or union elements, for the VPI member /
/// element objects of a frame variable, and the array's row stride; empty
/// for any other type.
fn aggregate_parts(ty: &sv_lang::Type<'_>, four_state: bool) -> (Vec<FramePart>, u32) {
    use sv_sim_sim::state::storage;
    let ty = ty.canonical();
    let width_of = |t: &sv_lang::Type<'_>, class: VarClass| match class {
        VarClass::Real => 64,
        VarClass::ShortReal => 32,
        _ => t.value_width(),
    };
    if ty.is_unpacked_struct() || (ty.is_unpacked_union() && !ty.is_tagged_union()) {
        let mut out = Vec::new();
        let Some(w) = storage::storage_width(&ty) else {
            return (Vec::new(), 0);
        };
        if !push_fields(&ty, (0, w as u32, 0), None, &mut out) {
            return (Vec::new(), 0);
        }
        return (out, 0);
    }
    let Some(shape) = crate::arrays::array_shape(&ty) else {
        return (Vec::new(), 0);
    };
    let elem = shape.elem.canonical();
    let agg = (elem.is_unpacked_struct() || elem.is_unpacked_union()) && !elem.is_tagged_union();
    if !agg && !((elem.is_integral() || elem.is_floating()) && !elem.is_unpacked_array()) {
        return (Vec::new(), 0);
    }
    let class = var_class_of(&elem);
    let w = if agg {
        // A row of a local array is `value_width` bits wide; the stored
        // struct must be exactly that (no string / handle fields).
        match storage::storage_width(&elem) {
            Some(w) if w == u64::from(elem.value_width()) => w as u32,
            _ => return (Vec::new(), 0),
        }
    } else {
        width_of(&elem, class)
    };
    let bits = if four_state { 2 * w } else { w };
    // A sub-array of a multi-dimensional array is a part of class `Array`
    // holding the next dimension's parts; rows are counted as in
    // `arrays::ArrayShape` (row-major, lowest index of each dimension first).
    struct Walk<'a, 'd> {
        shape: &'a crate::arrays::ArrayShape<'d>,
        leaf: (VarClass, u32, bool),
        stride: u32,
        elem: sv_lang::Type<'d>,
        agg: bool,
        ok: bool,
    }
    fn nodes(
        w: &mut Walk<'_, '_>,
        depth: usize,
        base_row: u64,
        parent: Option<u32>,
        out: &mut Vec<FramePart>,
    ) {
        let (l, r) = w.shape.dims[depth];
        for k in 0..w.shape.size(depth) {
            let index = l.min(r) + k as i64;
            let row = base_row + k * w.shape.stride(depth);
            let last = depth + 1 == w.shape.dims.len();
            let byte = if last { row as u32 * w.stride } else { 0 };
            out.push(FramePart {
                name: format!("[{index}]"),
                class: if last { w.leaf.0 } else { VarClass::Array },
                width: if last { w.leaf.1 } else { 0 },
                signed: last && w.leaf.2,
                offset: row as u32,
                index,
                parent,
                byte,
                cwidth: if last { w.leaf.1 } else { 0 },
                bit: 0,
                range: if last && !w.agg {
                    declared_range(&w.elem)
                } else {
                    None
                },
            });
            let me = (out.len() - 1) as u32;
            if !last {
                nodes(w, depth + 1, row, Some(me), out);
            } else if w.agg {
                let elem = w.elem;
                w.ok &= push_fields(&elem, (byte, w.leaf.1, 0), Some(me), out);
            }
        }
    }
    let mut walk = Walk {
        shape: &shape,
        leaf: (class, w, elem.is_signed()),
        stride: int_stride(bits),
        elem,
        agg,
        ok: true,
    };
    let mut parts = Vec::new();
    nodes(&mut walk, 0, 0, None, &mut parts);
    if !walk.ok {
        return (Vec::new(), 0);
    }
    (parts, walk.stride)
}

/// The element shape of the queue, dynamic array or associative array `ty`
/// when its elements are integral, real or string scalars (the ones VPI
/// reads and writes as values); `None` for any other type.
fn collection_of(ty: Option<&sv_lang::Type<'_>>) -> Option<sv_sim_exec::vpi_data::FrameColl> {
    use sv_sim_sim::collections::KeyKind;
    let canon = ty?.canonical();
    if !sv_sim_sim::state::storage::is_collection(&canon) {
        return None;
    }
    let elem = canon.element_type()?.canonical();
    if elem.is_unpacked_array() || !(elem.is_integral() || elem.is_floating() || elem.is_string()) {
        return None;
    }
    let width = match var_class_of(&elem) {
        VarClass::String | VarClass::Real => 64,
        VarClass::ShortReal => 32,
        _ => elem.value_width().max(1),
    };
    let key = if canon.is_associative_array() {
        match canon.associative_index_type() {
            Some(k) if k.is_string() => KeyKind::Str,
            Some(k) if k.is_integral() => KeyKind::Int {
                width: k.value_width().max(1),
                signed: k.is_signed(),
            },
            // A class-handle or other key: not modelled here.
            Some(_) => return None,
            None => KeyKind::Int {
                width: 64,
                signed: false,
            },
        }
    } else {
        KeyKind::Int {
            width: 32,
            signed: true,
        }
    };
    Some(sv_sim_exec::vpi_data::FrameColl {
        elem: var_class_of(&elem),
        width,
        signed: elem.is_signed(),
        assoc: canon.is_associative_array(),
        queue: canon.is_queue(),
        key,
    })
}

/// The frame site of `sub` with the automatic variables `vars` (one per
/// distinct subroutine and variable list).
fn frame_site_of(
    sub: &sv_lang::Symbol<'_>,
    vars: &[sv_lang::Symbol<'_>],
    refs: &[FrameRef],
    four_state: bool,
) -> u32 {
    frame_site_with(sub, frame_vars(vars, four_state), refs)
}

/// The [`FrameVar`] descriptions of `vars`.
fn frame_vars(vars: &[sv_lang::Symbol<'_>], four_state: bool) -> Vec<FrameVar> {
    vars.iter()
        .map(|v| {
            let ty = v.value_type();
            let class = ty.as_ref().map_or(VarClass::Logic, var_class_of);
            let mut scalar = ty
                .as_ref()
                .is_some_and(|t| (t.is_integral() || t.is_floating()) && !t.is_unpacked_array())
                && !matches!(class, VarClass::Event | VarClass::Class | VarClass::Chandle);
            let mut width = match class {
                VarClass::Real => 64,
                VarClass::ShortReal => 32,
                _ => ty.as_ref().map_or(1, |t| t.value_width()),
            };
            let (parts, stride) = ty
                .as_ref()
                .map_or((Vec::new(), 0), |t| aggregate_parts(t, four_state));
            // An unpacked struct of integral and real fields is one flat
            // vector (`storage::field_slice`), read in place like a packed one.
            if matches!(class, VarClass::Struct | VarClass::Union)
                && ty
                    .as_ref()
                    .is_some_and(|t| !t.canonical().is_packed_union())
                && !parts.is_empty()
                && let Some(w) = ty
                    .as_ref()
                    .and_then(sv_sim_sim::state::storage::storage_width)
            {
                scalar = true;
                width = w as u32;
            }
            FrameVar {
                id: var_id(&v.uid()),
                name: v.name().to_owned(),
                class,
                width,
                signed: ty.as_ref().is_some_and(sv_lang::Type::is_signed),
                four_state,
                scalar,
                parts,
                stride,
                coll: collection_of(ty.as_ref()),
                range: ty.as_ref().and_then(declared_range),
            }
        })
        .collect()
}

/// The frame site of `sub` with the variable descriptions `vars`.
fn frame_site_with(sub: &sv_lang::Symbol<'_>, vars: Vec<FrameVar>, refs: &[FrameRef]) -> u32 {
    let id = sub.uid();
    FRAME_SITES.with(|s| {
        let mut s = s.borrow_mut();
        if let Some(i) = s.iter().position(|f| {
            f.sub == id
                && f.vars.len() == vars.len()
                && f.vars.iter().zip(&vars).all(|(a, b)| a.name == b.name)
                && f.refs == refs
        }) {
            return i as u32;
        }
        s.push(FrameSite {
            sub: id,
            name: sub.name().to_owned(),
            vars,
            refs: refs.to_vec(),
        });
        (s.len() - 1) as u32
    })
}

/// The frame site number of an activation of `sub` with the automatic
/// variables `vars` (the one [`emit_frame_enter`] used for them).
pub(crate) fn frame_site(
    cx: &crate::expr::LowerCtx<'_, '_, '_>,
    sub: &sv_lang::Symbol<'_>,
    vars: &[sv_lang::Symbol<'_>],
) -> u32 {
    frame_site_of(
        sub,
        vars,
        &[],
        cx.repr == crate::value::ValueRepr::FourState,
    )
}

/// The number of the variable `id` ([`FrameVar::id`]).
fn var_id(id: &sv_sim_sim::symid::SymbolId) -> u32 {
    VAR_IDS.with(|m| {
        let mut m = m.borrow_mut();
        let n = m.len() as u32;
        *m.entry(id.clone()).or_insert(n)
    })
}

/// The rebind probe ([`OP_FRAME_REBIND`]) for the variables `vars` now
/// stored at the given addresses, at `b`'s position (its table in the
/// function's entry block). Nothing is emitted unless probes are enabled.
pub(crate) fn emit_frame_rebind<'ctx>(
    ctx: &'ctx inkwell::context::Context,
    module: &inkwell::module::Module<'ctx>,
    b: &inkwell::builder::Builder<'ctx>,
    vars: &[(
        sv_sim_sim::symid::SymbolId,
        inkwell::values::PointerValue<'ctx>,
    )],
) {
    if !enabled() || vars.is_empty() {
        return;
    }
    let i32t = ctx.i32_type();
    let i64t = ctx.i64_type();
    let f = b
        .get_insert_block()
        .and_then(|bb| bb.get_parent())
        .expect("the builder is inside a function");
    let entry = f.get_first_basic_block().expect("a function has an entry");
    let tb = ctx.create_builder();
    match entry.get_first_instruction() {
        Some(i) => tb.position_before(&i),
        None => tb.position_at_end(entry),
    }
    let arr_t = i64t.array_type(2 * vars.len() as u32);
    let table = tb.build_alloca(arr_t, "vpi_rebind").unwrap();
    for (k, (id, p)) in vars.iter().enumerate() {
        let at = |j: u64| {
            unsafe {
                b.build_in_bounds_gep(
                    arr_t,
                    table,
                    &[i32t.const_zero(), i32t.const_int(j, false)],
                    "",
                )
            }
            .unwrap()
        };
        b.build_store(
            at(2 * k as u64),
            i64t.const_int(u64::from(var_id(id)), false),
        )
        .unwrap();
        let addr = b.build_ptr_to_int(*p, i64t, "").unwrap();
        b.build_store(at(2 * k as u64 + 1), addr).unwrap();
    }
    let dyn_f = declare_dyn(ctx, module);
    let addr = b.build_ptr_to_int(table, i64t, "").unwrap();
    b.build_call(
        dyn_f,
        &[
            i32t.const_int(u64::from(OP_FRAME_REBIND), false).into(),
            i32t.const_int(vars.len() as u64, false).into(),
            addr.into(),
        ],
        "vpi_frame_rebind",
    )
    .unwrap();
}

/// Declares (once) and returns `sv_rt_vpi_dyn`.
fn dyn_fn<'ctx>(cx: &crate::expr::LowerCtx<'ctx, '_, '_>) -> inkwell::values::FunctionValue<'ctx> {
    declare_dyn(cx.ctx, cx.module)
}

/// Declares (once) `sv_rt_vpi_dyn` in `module`. Every probe kind must go
/// through here: the first declaration fixes the type, and a statement probe
/// reads the i32 result.
fn declare_dyn<'ctx>(
    ctx: &'ctx inkwell::context::Context,
    module: &inkwell::module::Module<'ctx>,
) -> inkwell::values::FunctionValue<'ctx> {
    let i32t = ctx.i32_type();
    let i64t = ctx.i64_type();
    module
        .get_function(SV_RT_VPI_DYN_SYMBOL)
        .unwrap_or_else(|| {
            module.add_function(
                SV_RT_VPI_DYN_SYMBOL,
                // Returns 1 when the shadowed signals must be reloaded.
                i32t.fn_type(&[i32t.into(), i32t.into(), i64t.into()], false),
                Some(inkwell::module::Linkage::External),
            )
        })
}

/// The frame-entry probe of an activation of `sub` whose automatic variables
/// are `vars` (their storage is looked up in `cx.locals`), at the builder's
/// position. Returns the frame site number for the matching
/// [`emit_frame_exit`]; `None` when probes are off.
pub(crate) fn emit_frame_enter(
    cx: &crate::expr::LowerCtx<'_, '_, '_>,
    sub: &sv_lang::Symbol<'_>,
    vars: &[sv_lang::Symbol<'_>],
    refs: &[FrameRef],
) -> Option<u32> {
    if !enabled() || crate::stmt_shared::current_block_terminated(cx) {
        return None;
    }
    let site = frame_site_of(
        sub,
        vars,
        refs,
        cx.repr == crate::value::ValueRepr::FourState,
    );
    // The functions' `ref` formals with a working copy follow the variables
    // in the table ([`FrameRef::store`]).
    let mut slots: Vec<sv_lang::Symbol<'_>> = vars.to_vec();
    slots.extend(
        sub.subroutine_arguments()
            .filter(|f| refs.iter().any(|r| r.store.is_some() && r.name == f.name())),
    );
    let table = storage_table(cx, &slots);
    let i32t = cx.ctx.i32_type();
    cx.b.build_call(
        dyn_fn(cx),
        &[
            i32t.const_int(u64::from(OP_FRAME_ENTER), false).into(),
            i32t.const_int(u64::from(site), false).into(),
            table.into(),
        ],
        "vpi_frame_enter",
    )
    .unwrap();
    crate::dpi::reload_shadows(cx);
    Some(site)
}

/// A table of the storage pointers of `vars` (looked up in `cx.locals`; NULL
/// where a variable has none), as the `i64` address the probes take.
fn storage_table<'ctx>(
    cx: &crate::expr::LowerCtx<'ctx, '_, '_>,
    vars: &[sv_lang::Symbol<'_>],
) -> inkwell::values::IntValue<'ctx> {
    let ptr_t = cx.ctx.ptr_type(inkwell::AddressSpace::default());
    if vars.is_empty() {
        cx.ctx.i64_type().const_zero()
    } else {
        // The table lives in the entry block's frame: one slot per variable.
        let f =
            cx.b.get_insert_block()
                .and_then(|bb| bb.get_parent())
                .expect("the builder is inside a function");
        let entry = f.get_first_basic_block().expect("a function has an entry");
        let tb = cx.ctx.create_builder();
        match entry.get_first_instruction() {
            Some(i) => tb.position_before(&i),
            None => tb.position_at_end(entry),
        }
        let arr_t = ptr_t.array_type(vars.len() as u32);
        let table = tb.build_alloca(arr_t, "vpi_frame_vars").unwrap();
        for (k, v) in vars.iter().enumerate() {
            let slot = unsafe {
                cx.b.build_in_bounds_gep(
                    arr_t,
                    table,
                    &[
                        cx.ctx.i32_type().const_zero(),
                        cx.ctx.i32_type().const_int(k as u64, false),
                    ],
                    "vpi_frame_var",
                )
            }
            .unwrap();
            let p = cx
                .locals
                .get(&v.uid())
                .map_or(ptr_t.const_null(), |&(p, _, _)| p);
            cx.b.build_store(slot, p).unwrap();
        }
        cx.b.build_ptr_to_int(table, cx.ctx.i64_type(), "vpi_frame_table")
            .unwrap()
    }
}

/// `base` followed by the select segment `seg` (`[1]` or `.f`). A static path
/// just concatenates; an automatic's marker (`@auto:a`) separates the variable
/// from the select path of the member or element with `|`, once (37.15 detail
/// 3: the runtime resolves the variable in the calling frame, then the part).
fn join_actual_path(base: &str, seg: &str) -> String {
    let auto = base.starts_with(sv_sim_exec::vpi_data::AUTO_ACTUAL);
    if auto && !base.contains('|') {
        format!("{base}|{seg}")
    } else {
        format!("{base}{seg}")
    }
}

/// The root variable of a select / member chain `actual` (the variable itself
/// for a plain name).
fn select_root(actual: sv_lang::Expression<'_>) -> sv_lang::Expression<'_> {
    use sv_lang::kinds::ExpressionKind as K;
    let mut e = actual;
    loop {
        let kind = e.kind();
        let next = if matches!(kind, K::ElementSelect) {
            e.select_value()
        } else if matches!(kind, K::MemberAccess) {
            e.left()
        } else {
            None
        };
        match next {
            Some(n) => e = n,
            None => return e,
        }
    }
}

/// The static path of the object a `ref` actual names, `""` when it is not a
/// plain static variable or net (an automatic, a select, a formal passed on).
fn ref_actual_path(actual: sv_lang::Expression<'_>) -> String {
    use sv_lang::kinds::ExpressionKind as K;
    // 37.15 detail 3: a constant element select of a static fixed-size array
    // names the element object (`top.arr[1]`).
    if actual.kind() == K::ElementSelect {
        let elem = (|| {
            let base = actual.select_value()?;
            let fixed = base.expr_type().is_some_and(|t| {
                t.is_unpacked_array() && !sv_sim_sim::state::storage::is_collection(&t)
            });
            let i = actual.selector()?.constant_value()?.as_i64()?;
            let b = ref_actual_path(base);
            (fixed && !b.is_empty()).then(|| join_actual_path(&b, &format!("[{i}]")))
        })();
        return elem.unwrap_or_default();
    }
    // 37.15 detail 3 / 37.26: a member of a static unpacked struct names the
    // struct-member object (`top.st.f`), however deeply nested.
    if actual.kind() == K::MemberAccess {
        let name = (|| {
            let base = actual.left()?;
            let bt = base.expr_type()?.canonical();
            let member = actual.member_symbol()?;
            let b = ref_actual_path(base);
            (bt.is_unpacked_struct() && !b.is_empty())
                .then(|| join_actual_path(&b, &format!(".{}", member.name())))
        })();
        return name.unwrap_or_default();
    }
    if !sv_sim_sim::symid::is_value_name(actual.kind()) {
        return String::new();
    }
    match actual.referenced_symbol() {
        Some(sym)
            if sym.kind() == sv_lang::kinds::SymbolKind::Net
                || (sym.kind() == sv_lang::kinds::SymbolKind::Variable
                    && sym.variable_lifetime() == sv_lang::VariableLifetime::Static) =>
        {
            sym.hierarchical_path()
        }
        // 37.15 detail 3: an automatic variable of the caller's frame; the
        // runtime resolves the marker against the calling frame at the call.
        Some(sym)
            if sym.kind() == sv_lang::kinds::SymbolKind::Variable
                && sym.variable_lifetime() == sv_lang::VariableLifetime::Automatic =>
        {
            format!("{}{}", sv_sim_exec::vpi_data::AUTO_ACTUAL, sym.name())
        }
        // 37.15 detail 3: a `ref` formal passed on names whatever the calling
        // frame's formal is bound to; the runtime resolves it at the call.
        Some(sym)
            if sym.kind() == sv_lang::kinds::SymbolKind::FormalArgument
                && sym.formal_argument_direction() == sv_lang::ArgumentDirection::Ref =>
        {
            format!("{}{}", sv_sim_exec::vpi_data::AUTO_ACTUAL, sym.name())
        }
        _ => String::new(),
    }
}

/// The `ref` formals of `callee` as frame placeholders (37.15 detail 1): a
/// function's frame lists them as ref objs ahead of its other automatics, each
/// bound by the call ([`emit_ref_bind`]).
pub(crate) fn ref_formal_placeholders(
    cx: &crate::expr::LowerCtx<'_, '_, '_>,
    callee: &sv_lang::Symbol<'_>,
) -> Vec<FrameRef> {
    // A function works on a copy of each `ref` formal (the callee's storage);
    // a value access while the body runs must reach it, not the actual
    // (37.15 detail 1). A body that carries live element references
    // (`takes_elem_refs`) refreshes the copy through them, so it keeps the
    // refusal.
    let live = !crate::func::takes_elem_refs(callee);
    let four_state = cx.repr == crate::value::ValueRepr::FourState;
    callee
        .subroutine_arguments()
        .filter(|f| f.formal_argument_direction() == sv_lang::ArgumentDirection::Ref)
        .map(|f| FrameRef {
            name: f.name().to_owned(),
            actual: String::new(),
            store: (live && ref_formal_has_store(cx, &f))
                .then(|| frame_vars(&[f], four_state).remove(0)),
        })
        .collect()
}

/// Whether `ref` formal `f` of the function being lowered has storage the
/// VPI side can read in place (a packed or real scalar, or an unpacked array
/// or struct of them; not a collection or a wider-than-65535-bit image).
fn ref_formal_has_store(cx: &crate::expr::LowerCtx<'_, '_, '_>, f: &sv_lang::Symbol<'_>) -> bool {
    // A static function's formal is the instance's signal, not the call's own
    // storage: not read in place here, so it stays refused by name.
    !sv_sim_sim::netlist::is_static_procedural_var(f)
        && f.variable_lifetime() != sv_lang::VariableLifetime::Static
        && cx.locals.contains_key(&f.uid())
        && f.value_type().is_some_and(|t| {
            !sv_sim_sim::state::storage::is_collection(&t) && !crate::huge::is_image_type(&t)
        })
}

/// Before a call of `callee`, records which objects its `ref` formals are bound
/// to ([`OP_REF_BIND`]); a callee without `ref` formals, or probes off, emit
/// nothing. `actuals` pair 1:1 with `callee`'s formals.
pub(crate) fn emit_ref_bind<'d>(
    cx: &crate::expr::LowerCtx<'_, '_, '_>,
    callee: &sv_lang::Symbol<'d>,
    actuals: &[sv_lang::Expression<'d>],
) {
    if !enabled() {
        return;
    }
    let mut refs: Vec<FrameRef> = Vec::new();
    let mut ref_actuals: Vec<sv_lang::Expression<'d>> = Vec::new();
    for (f, a) in callee.subroutine_arguments().zip(actuals) {
        if f.formal_argument_direction() == sv_lang::ArgumentDirection::Ref {
            refs.push(FrameRef {
                name: f.name().to_owned(),
                actual: ref_actual_path(*a),
                store: None,
            });
            ref_actuals.push(*a);
        }
    }
    emit_bind(cx, callee, refs, &ref_actuals);
}

/// The path a `ref` actual binds to ([`ref_actual_path`]), for a call that
/// copies the formal in and out (an inlined task, 37.15 detail 3).
pub(crate) fn ref_actual_of(actual: sv_lang::Expression<'_>) -> String {
    ref_actual_path(actual)
}

/// Records `refs` (the bindings of `callee`'s `ref` formals) for the frame
/// about to be entered ([`OP_REF_BIND`]); `ref_actuals` are the actuals whose
/// automatic variables the runtime must be handed by storage.
pub(crate) fn emit_bind<'d>(
    cx: &crate::expr::LowerCtx<'_, '_, '_>,
    callee: &sv_lang::Symbol<'d>,
    refs: Vec<FrameRef>,
    ref_actuals: &[sv_lang::Expression<'d>],
) {
    if refs.is_empty() || !enabled() {
        return;
    }
    // 37.15 detail 3: an automatic variable actual (a block local of a
    // frameless procedural block has no frame of its own) is handed to the
    // runtime by storage, so a call whose caller has no frame holding it can
    // still name it.
    let mut autos: Vec<sv_lang::Symbol<'d>> = Vec::new();
    for a in ref_actuals {
        // An element or member of an automatic aggregate hands over the
        // aggregate's storage (the actual's path names the part).
        let a = &select_root(*a);
        if sv_sim_sim::symid::is_value_name(a.kind())
            && let Some(sym) = a.referenced_symbol()
            && sym.kind() == sv_lang::kinds::SymbolKind::Variable
            && sym.variable_lifetime() == sv_lang::VariableLifetime::Automatic
            && cx.locals.contains_key(&sym.uid())
            && !autos.iter().any(|x| x.uid() == sym.uid())
        {
            autos.push(sym);
        }
    }
    let vars = frame_vars(&autos, cx.repr == crate::value::ValueRepr::FourState);
    let site = FRAME_SITES.with(|s| {
        let mut s = s.borrow_mut();
        let id = callee.uid();
        if let Some(i) = s
            .iter()
            .position(|f| f.sub == id && f.vars == vars && f.refs == refs && f.name.is_empty())
        {
            return i as u32;
        }
        s.push(FrameSite {
            sub: id,
            name: String::new(),
            vars,
            refs,
        });
        (s.len() - 1) as u32
    });
    let table = storage_table(cx, &autos);
    if crate::stmt_shared::current_block_terminated(cx) {
        return;
    }
    let i32t = cx.ctx.i32_type();
    cx.b.build_call(
        dyn_fn(cx),
        &[
            i32t.const_int(u64::from(OP_REF_BIND), false).into(),
            i32t.const_int(u64::from(site), false).into(),
            table.into(),
        ],
        "vpi_ref_bind",
    )
    .unwrap();
}

/// The frame-exit probe of frame site `site` at the builder's position (which
/// may be just before a block's terminator).
pub(crate) fn emit_frame_exit(cx: &crate::expr::LowerCtx<'_, '_, '_>, site: u32) {
    let i32t = cx.ctx.i32_type();
    cx.b.build_call(
        dyn_fn(cx),
        &[
            i32t.const_int(u64::from(OP_FRAME_EXIT), false).into(),
            i32t.const_int(u64::from(site), false).into(),
            cx.ctx.i64_type().const_zero().into(),
        ],
        "vpi_frame_exit",
    )
    .unwrap();
}

/// Puts a frame-exit probe of `site` before every `ret` of `f`.
pub(crate) fn emit_frame_exits_before_returns(
    cx: &crate::expr::LowerCtx<'_, '_, '_>,
    f: inkwell::values::FunctionValue<'_>,
    site: u32,
) {
    let saved = cx.b.get_insert_block();
    for bb in f.get_basic_block_iter() {
        if let Some(t) = bb.get_terminator()
            && t.get_opcode() == inkwell::values::InstructionOpcode::Return
        {
            cx.b.position_before(&t);
            emit_frame_exit(cx, site);
        }
    }
    if let Some(bb) = saved {
        cx.b.position_at_end(bb);
    }
}

/// The site number of `st` (one per distinct statement).
fn site_of(st: &Statement<'_>) -> u32 {
    let key = key_of(st);
    STMT_SITES.with(|s| {
        let mut s = s.borrow_mut();
        if let Some(&n) = s.1.get(&key) {
            return n;
        }
        let n = s.0.len() as u32;
        s.0.push(key);
        s.1.insert(key, n);
        n
    })
}

/// Emits the call of `sv_rt_vpi_dyn(op, a, b)` at the builder's position, and
/// reloads the process's shadows: the callback may have written any signal.
pub(crate) fn emit_dyn_call(cx: &crate::expr::LowerCtx<'_, '_, '_>, op: u32, a: u32, b: u64) {
    // Dead code after a `return`/`break`: nothing runs there, and nothing may
    // follow a terminator.
    if crate::stmt_shared::current_block_terminated(cx) {
        return;
    }
    let i32t = cx.ctx.i32_type();
    let i64t = cx.ctx.i64_type();
    let f = dyn_fn(cx);
    let call =
        cx.b.build_call(
            f,
            &[
                i32t.const_int(op as u64, false).into(),
                i32t.const_int(a as u64, false).into(),
                i64t.const_int(b, false).into(),
            ],
            "vpi_dyn",
        )
        .unwrap();
    // A statement probe answers whether a user callback ran: only then can a
    // signal have changed under the process, so only then are its shadows
    // re-read (a cocotb run has a probe at every statement and a shadow per
    // signal the process touches, and no callback at nearly all of them).
    if op != OP_STMT
        || !cx
            .write_through
            .as_ref()
            .is_some_and(|wt| !wt.signals.is_empty())
    {
        crate::dpi::reload_shadows(cx);
        return;
    }
    let Some(parent) = cx.b.get_insert_block().and_then(|bb| bb.get_parent()) else {
        crate::dpi::reload_shadows(cx);
        return;
    };
    let ran = call.try_as_basic_value().unwrap_basic().into_int_value();
    let fired =
        cx.b.build_int_compare(
            inkwell::IntPredicate::NE,
            ran,
            i32t.const_zero(),
            "vpi_dyn_ran",
        )
        .unwrap();
    let cont_bb = cx.ctx.append_basic_block(parent, "vpi_cont");
    let wt = cx.write_through.as_ref().expect("checked above");
    let probe_bb = cx.b.get_insert_block().expect("builder positioned");
    let mut shared = wt.reload.borrow_mut();
    // One re-read block per function, not per probe: a probe per statement
    // times a shadow per signal was quadratic IR (77-106k lines per always
    // block in verilog-pcie, most of the JIT time). Every probe branches
    // there with its own number and the switch ending the block returns to
    // that probe's continuation.
    let mut sh = match *shared {
        Some(sh) if sh.func == parent => sh,
        _ => {
            let bb = cx.ctx.append_basic_block(parent, "vpi_reload");
            cx.b.position_at_end(bb);
            let phi = cx.b.build_phi(i32t, "vpi_ret").unwrap();
            crate::dpi::reload_shadows(cx);
            let sw =
                cx.b.build_switch(phi.as_basic_value().into_int_value(), cont_bb, &[])
                    .unwrap();
            SharedReload {
                func: parent,
                bb,
                phi,
                sw,
                next: 0,
            }
        }
    };
    let kv = i32t.const_int(sh.next as u64, false);
    sh.next += 1;
    *shared = Some(sh);
    sh.phi.add_incoming(&[(&kv, probe_bb)]);
    // SAFETY: `sw` is the switch of the shared block (built above or by an
    // earlier probe), `kv` an i32 constant and `cont_bb` a block of the same
    // function.
    unsafe {
        llvm_sys::core::LLVMAddCase(
            inkwell::values::AsValueRef::as_value_ref(&sh.sw),
            inkwell::values::AsValueRef::as_value_ref(&kv),
            cont_bb.as_mut_ptr(),
        );
    }
    cx.b.position_at_end(probe_bb);
    cx.b.build_conditional_branch(fired, sh.bb, cont_bb)
        .unwrap();
    cx.b.position_at_end(cont_bb);
}

/// The shared re-read block of one function's statement probes: it reloads
/// every shadow, then a `switch` on the phi of the probe numbers returns to the
/// probe's own continuation.
#[derive(Clone, Copy)]
pub struct SharedReload<'ctx> {
    func: inkwell::values::FunctionValue<'ctx>,
    bb: inkwell::basic_block::BasicBlock<'ctx>,
    phi: inkwell::values::PhiValue<'ctx>,
    sw: inkwell::values::InstructionValue<'ctx>,
    next: u32,
}

/// A `cbStmt` probe for `st`, unconditionally (the loops use it for their
/// per-iteration points). Nothing is emitted unless probes are enabled.
pub(crate) fn emit_stmt_probe(cx: &crate::expr::LowerCtx<'_, '_, '_>, st: &Statement<'_>) {
    if enabled() {
        emit_dyn_call(cx, OP_STMT, site_of(st), 0);
    }
    crate::stmtcov::emit_probe(cx, st);
}

/// The `cbDisable` probe of the `disable` statement `st`, which has just taken
/// effect (the named block it targets is disabled). Nothing is emitted unless
/// probes are enabled, or when the target cannot be read.
pub(crate) fn emit_disable_probe(cx: &crate::expr::LowerCtx<'_, '_, '_>, st: &Statement<'_>) {
    if !enabled() {
        return;
    }
    // A hierarchical `a.b` target reports its last segment (the block).
    let label = match sv_sim_sim::process::cfg::disable_label(st) {
        Ok(l) => l,
        Err(_) => {
            match sv_sim_sim::process::cfg::disable_path(st).and_then(|p| p.last().cloned()) {
                Some(l) => l,
                None => return,
            }
        }
    };
    let key = key_of(st);
    let site = DISABLE_SITES.with(|s| {
        let mut s = s.borrow_mut();
        if let Some(i) = s.iter().position(|(k, l)| *k == key && *l == label) {
            return i as u32;
        }
        s.push((key, label));
        (s.len() - 1) as u32
    });
    emit_dyn_call(cx, OP_DISABLE, site, 0);
}

/// The process lowering is about to lower a statement of a segment's body:
/// the CFG recorded that statement's entry probe itself
/// ([`sv_sim_sim::process::cfg::BasicBlock::probes`]), so the statement
/// lowering must not emit it a second time.
pub(crate) fn mark_probed_by_cfg() {
    PROBED_BY_CFG.with(|p| p.set(true));
}

/// The entry probe of a statement the lowering visits itself (Table 38-6's
/// "before the statement executes"): emitted unless the CFG already did (see
/// [`mark_probed_by_cfg`]). Always consumes the mark, so it only ever applies
/// to the statement lowered right after it was set.
pub(crate) fn emit_entry_probe(cx: &crate::expr::LowerCtx<'_, '_, '_>, st: &Statement<'_>) {
    let by_cfg = PROBED_BY_CFG.with(|p| p.replace(false));
    if !by_cfg && sv_sim_sim::process::cfg::is_entry_probe_kind(st) {
        if enabled() {
            emit_dyn_call(cx, OP_STMT, site_of(st), 0);
        }
        crate::stmtcov::emit_probe(cx, st);
    }
}