vj-codegen 0.2.0

//! Vectors wider than [`WIDE_THRESHOLD`] bits (LRM 1800-2023 6.9.1).
//!
//! A narrow value is one LLVM `i(2N)` (`(unk << N) | val`). Above
//! `WIDE_THRESHOLD` that integer type is not usable: LLVM spends tens of
//! seconds legalising a single shift or add of an `i131070`. A huge value is
//! instead an image in memory -- the `PK` layout of
//! [`vj_runtime::wide_rt`] (the bytes of the packed `i(2N)`: value plane
//! at bit 0, unknown plane at bit N, little-endian `u64` words) -- and every
//! operator is one call of `sv_rt_wide`, which reads its operands into owned
//! planes before it writes the destination (so a destination may alias an
//! operand).
//!
//! * [`HugeValue`] is such an image: a pointer plus a width and a signedness.
//!   A module signal's process shadow is a `[words x i64]` `alloca`, and the
//!   value of a named reference is that very pointer (no copy).
//! * Temporaries come from a pool of entry-block `alloca`s whose state lives in
//!   their IR names (`hugetmp_free_{words}_` / `hugetmp_busy_{words}_`), so a
//!   result is released as soon as its consumer has read it and the frame
//!   stays at the high-water mark of live values instead of one temporary per
//!   operator. A missed release only grows the frame; a release of a pointer
//!   that is not a busy temporary (a shadow, a constant) is a no-op.
//! * A select (`a[i]`, `a[hi:lo]`, `a[i +: n]`, `s.field`, nested) is a chain
//!   of windows: a signed bit offset plus an "invalid" flag per level. Reads
//!   fold the offsets and extract once; writes read-modify-write through the
//!   chain (an index that can straddle the edge of its parent, i.e. a
//!   multi-element dynamic or an out-of-range constant part-select, makes the
//!   parent window itself a value to extract, patch and insert back).
//! * A narrow result of a huge operand (a compare, a reduction, `!`, a select
//!   narrower than the threshold) is loaded back into an ordinary
//!   [`PackedValue`], and a narrow operand of a huge operation is spilled to a
//!   temporary image first.
//!
//! The width at which this path starts is [`DEFAULT_THRESHOLD`] (2048); the
//! knob `VJ_HUGE_FROM_BITS=<64..=65535>` moves it (64 re-runs the whole
//! narrow-path corpus through this code, the differential test in
//! `vj/tests/huge_vec.rs`; 65535 restores integer lowering up to its
//! ceiling). Structural types (function arguments, returns and locals,
//! unpacked-array elements, DPI) stay integers up to `WIDE_THRESHOLD`
//! regardless of the knob.
//!
//! Native here (one `sv_rt_wide` call or a few, four-state per LRM clause 11):
//! every unary/binary operator including `**` (any exponent width), `/`, `%`
//! and the shifts; compares, reductions and `!`; concatenation and
//! replication; selects of every kind, read and write; `++`/`--`; `?:`
//! (branching when an arm has side effects); `case`/`casex`/`casez`, `case ...
//! inside` and the `inside`/`dist` operators; `$signed`/`$unsigned`, the
//! popcount family and `$clog2`; conversions to and from `real`, `shortreal`
//! and `string`; streaming (both sides) and bit-stream casts.
//!
//! What an operation does not have a native form for falls back
//! ([`fall_back`]) to the ordinary integer lowering while every width involved
//! fits `WIDE_THRESHOLD`: correct, but it pays that lowering's compile cost.
//! Above `WIDE_THRESHOLD`, each of these is a loud [`LoweringError`]: huge
//! function arguments/returns/locals and DPI arguments above 128 bits,
//! unpacked-array elements of huge width, streaming of aggregates and
//! unpacked/dynamic arrays, a nonblocking assignment outside a process
//! statement, a pattern-matching `?:`, and a two-state (`ValueRepr::TwoState`)
//! build (the in-memory form is four-state only).

use std::hash::{Hash, Hasher};
use std::sync::OnceLock;

use inkwell::AddressSpace;
use inkwell::IntPredicate;
use inkwell::basic_block::BasicBlock;
use inkwell::module::Linkage;
use inkwell::values::{FunctionValue, InstructionOpcode, IntValue, PointerValue};
use sv_lang::{BinaryOp, Bit, UnaryOp, kinds::ExpressionKind};
use vj_runtime::wide_rt::{self, WideOp};
use vj_sim::symid::SymbolUid;

use crate::error::LoweringError;
use crate::expr::{
    LowerCtx, LoweredValue, lower_expr_h, lower_expr_regular, require_const_i64,
    write_through_commit, write_through_store,
};
use crate::ops::OpBuilder;
use crate::select::{SelectGeom, checked_window_width, field_window, geom_of};
use crate::types::TypeWidth;
use crate::value::{PackedValue, ValueRepr, WIDE_THRESHOLD};

/// The debug knob that lowers the huge threshold (see the module docs).
pub const THRESHOLD_ENV: &str = "VJ_HUGE_FROM_BITS";

/// The default width above which an integral value is a [`HugeValue`].
///
/// Measured (`bench/huge/threshold.py`): the integer lowering costs 0.19 s of
/// startup at 2048 bits, 0.57 s at 4096 and 1.84 s at 8192 (and minutes per
/// multiply or divide near 65535), while the in-memory form starts in about
/// 0.03 s at any width and costs 2-4x per cheap operation (an add is ~1.2 us
/// against ~0.6 us at 1024 bits). Up to 2048 bits the integer path never loses
/// by more than its 0.19 s startup; above it, every non-trivial workload
/// (more than ~1e5 operations, or any multiply/divide/modulo) is faster huge.
pub const DEFAULT_THRESHOLD: u32 = 2048;

/// The width above which an integral value is a [`HugeValue`]:
/// [`DEFAULT_THRESHOLD`] unless [`THRESHOLD_ENV`] sets it (`64..=WIDE_THRESHOLD`;
/// `WIDE_THRESHOLD` restores the pure-integer behaviour up to its ceiling). An
/// invalid knob is a loud panic.
pub fn threshold() -> u32 {
    static T: OnceLock<u32> = OnceLock::new();
    *T.get_or_init(|| match std::env::var(THRESHOLD_ENV) {
        Err(_) => DEFAULT_THRESHOLD,
        Ok(s) => match s.trim().parse::<u32>() {
            Ok(n) if (64..=WIDE_THRESHOLD).contains(&n) => n,
            _ => panic!("{THRESHOLD_ENV}={s:?}: expected an integer in 64..={WIDE_THRESHOLD}"),
        },
    })
}

/// Whether a value of `width` bits is a [`HugeValue`].
pub fn is_huge_width(width: u32) -> bool {
    width > threshold()
}

/// Whether `ty` is an integral type wide enough to be a [`HugeValue`].
pub fn is_huge_type(ty: &sv_lang::Type<'_>) -> bool {
    ty.is_integral() && !ty.is_floating() && is_huge_width(ty.value_width())
}

/// 64-bit words in the `PK` image of a `width`-bit value (`2 * width` bits).
pub fn pk_words(width: u32) -> u32 {
    (2 * u64::from(width)).div_ceil(64) as u32
}

/// A value in memory: the `PK` image at `ptr` (`pk_words(width)` words).
#[derive(Clone, Copy)]
pub struct HugeValue<'ctx> {
    pub ptr: PointerValue<'ctx>,
    pub width: u32,
    pub signed: bool,
}

impl HugeValue<'_> {
    /// The `sv_rt_wide` operand-info word of this value.
    pub fn info(self) -> u64 {
        wide_rt::info(self.width, self.signed)
    }
}

fn loud<T>(cx: &LowerCtx<'_, '_, '_>, what: &str) -> Result<T, LoweringError> {
    Err(LoweringError::unsupported_at(what, &cx.fn_sym))
}

// ---------------------------------------------------------------------------
// Temporary pool

const FREE: &str = "hugetmp_free_";
const BUSY: &str = "hugetmp_busy_";

fn entry_block<'ctx>(cx: &LowerCtx<'ctx, '_, '_>) -> BasicBlock<'ctx> {
    cx.b.get_insert_block()
        .and_then(|bb| bb.get_parent())
        .and_then(|f| f.get_first_basic_block())
        .expect("lowering happens inside a function with an entry block")
}

/// A `words`-word temporary: a free pooled `alloca` of that size if the entry
/// block has one, else a new one. Valid until [`release`]d.
pub(crate) fn acquire<'ctx>(cx: &LowerCtx<'ctx, '_, '_>, words: u32) -> PointerValue<'ctx> {
    let free = format!("{FREE}{words}_");
    let busy = format!("{BUSY}{words}_");
    let mut cur = entry_block(cx).get_first_instruction();
    while let Some(inst) = cur {
        if inst.get_opcode() != InstructionOpcode::Alloca {
            break;
        }
        if inst
            .get_name()
            .is_some_and(|n| n.to_bytes().starts_with(free.as_bytes()))
        {
            let p = PointerValue::try_from(inst).expect("an alloca is a pointer");
            p.set_name(&busy);
            return p;
        }
        cur = inst.get_next_instruction();
    }
    let ty = cx.ctx.i64_type().array_type(words);
    crate::expr::entry_alloca(cx, ty, &busy)
}

/// A pooled temporary for a `width`-bit value.
fn temp<'ctx>(cx: &LowerCtx<'ctx, '_, '_>, width: u32) -> PointerValue<'ctx> {
    acquire(cx, pk_words(width))
}

/// Returns a busy pooled temporary to the pool; anything else (a shadow, a
/// constant, an already-free temporary) is left alone.
pub(crate) fn release(ptr: PointerValue<'_>) {
    let Some(inst) = ptr.as_instruction() else {
        return;
    };
    if inst.get_opcode() != InstructionOpcode::Alloca {
        return;
    }
    let Some(name) = inst.get_name().and_then(|n| n.to_str().ok()) else {
        return;
    };
    let Some(rest) = name.strip_prefix(BUSY) else {
        return;
    };
    let digits: String = rest.chars().take_while(char::is_ascii_digit).collect();
    ptr.set_name(&format!("{FREE}{digits}_"));
}

/// The `words` encoded in a pool name (`hugetmp_busy_12_3` -> 12), for tests.
#[cfg(test)]
fn pool_words(name: &str) -> Option<u32> {
    let rest = name
        .strip_prefix(BUSY)
        .or_else(|| name.strip_prefix(FREE))?;
    rest.chars()
        .take_while(char::is_ascii_digit)
        .collect::<String>()
        .parse()
        .ok()
}

/// The runtime entry for a huge signal's process shadow
/// (`vj_sim::sched::real_sv_rt_huge`).
pub const SV_RT_HUGE_SYMBOL: &str = "sv_rt_huge";

/// `void* sv_rt_huge(u32 op, SimHandle, u32 slot, u32 width, i64 a, i64 b,
/// const u64* p, u64 delay)`, fetched or declared in the module being lowered.
fn rt_huge_fn<'ctx>(cx: &LowerCtx<'ctx, '_, '_>) -> FunctionValue<'ctx> {
    cx.module
        .get_function(SV_RT_HUGE_SYMBOL)
        .unwrap_or_else(|| {
            let ptr = cx.ctx.ptr_type(AddressSpace::default());
            let i32t = cx.ctx.i32_type();
            let i64t = cx.ctx.i64_type();
            let ty = ptr.fn_type(
                &[
                    i32t.into(),
                    ptr.into(),
                    i32t.into(),
                    i32t.into(),
                    i64t.into(),
                    i64t.into(),
                    ptr.into(),
                    i64t.into(),
                ],
                false,
            );
            cx.module
                .add_function(SV_RT_HUGE_SYMBOL, ty, Some(Linkage::External))
        })
}

#[allow(clippy::too_many_arguments)]
fn rt_huge<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    op: u32,
    handle: PointerValue<'ctx>,
    slot: u32,
    width: u32,
    a: IntValue<'ctx>,
    b: IntValue<'ctx>,
    p: PointerValue<'ctx>,
    delay: IntValue<'ctx>,
) -> PointerValue<'ctx> {
    let i32t = cx.ctx.i32_type();
    cx.b.build_call(
        rt_huge_fn(cx),
        &[
            i32t.const_int(u64::from(op), false).into(),
            handle.into(),
            i32t.const_int(u64::from(slot), false).into(),
            i32t.const_int(u64::from(width), false).into(),
            a.into(),
            b.into(),
            p.into(),
            delay.into(),
        ],
        "rt_huge_call",
    )
    .unwrap()
    .try_as_basic_value()
    .unwrap_basic()
    .into_pointer_value()
}

/// The long-lived PK image (the process shadow) of the huge module signal in
/// `slot` of `handle`, current (`state::huge_prime`). `rebuild` re-reads it
/// from STATE even if the mirror believes it is current.
pub(crate) fn rt_prime<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    handle: PointerValue<'ctx>,
    slot: u32,
    width: u32,
    rebuild: bool,
) -> PointerValue<'ctx> {
    let i64t = cx.ctx.i64_type();
    rt_huge(
        cx,
        vj_sim::sched::HUGE_OP_PRIME,
        handle,
        slot,
        width,
        i64t.const_int(u64::from(rebuild), false),
        i64t.const_zero(),
        null_ptr(cx),
        i64t.const_zero(),
    )
}

/// Widens the dirty window of the shadow of `(handle, slot)` by the value bits
/// `[start, start + len)` (clipped to the signal; a `len <= 0` marks nothing).
pub(crate) fn rt_mark<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    handle: PointerValue<'ctx>,
    slot: u32,
    width: u32,
    start: IntValue<'ctx>,
    len: IntValue<'ctx>,
) {
    rt_huge(
        cx,
        vj_sim::sched::HUGE_OP_MARK,
        handle,
        slot,
        width,
        start,
        len,
        null_ptr(cx),
        cx.ctx.i64_type().const_zero(),
    );
}

/// Commits the dirty window of the shadow of `(handle, slot)` to STATE
/// (`state::huge_commit`); a clean shadow is a no-op.
pub(crate) fn rt_commit<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    handle: PointerValue<'ctx>,
    slot: u32,
    width: u32,
) {
    let i64t = cx.ctx.i64_type();
    rt_huge(
        cx,
        vj_sim::sched::HUGE_OP_COMMIT,
        handle,
        slot,
        width,
        i64t.const_zero(),
        i64t.const_zero(),
        null_ptr(cx),
        i64t.const_zero(),
    );
}

/// Stages the nonblocking write `sig[start +: len] <= image` of the huge module
/// signal in `slot` of `handle` (`state::huge_nba_window`): `image` is the PK
/// image of the `len`-bit right-hand side, `delay` its intra-assignment delay.
/// Only the plane words the select covers cross the runtime boundary.
#[allow(clippy::too_many_arguments)]
fn rt_nba_window<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    handle: PointerValue<'ctx>,
    slot: u32,
    width: u32,
    start: IntValue<'ctx>,
    len: u32,
    image: PointerValue<'ctx>,
    delay: IntValue<'ctx>,
) {
    rt_huge(
        cx,
        vj_sim::sched::HUGE_OP_NBA,
        handle,
        slot,
        width,
        start,
        cx.ctx.i64_type().const_int(u64::from(len), false),
        image,
        delay,
    );
}

fn null_ptr<'ctx>(cx: &LowerCtx<'ctx, '_, '_>) -> PointerValue<'ctx> {
    cx.ctx.ptr_type(AddressSpace::default()).const_null()
}

/// A process shadow for a huge module signal: `pk_words(width)` words in the
/// entry block, never pooled.
pub(crate) fn shadow_alloca<'ctx>(
    ctx: &'ctx inkwell::context::Context,
    b: &inkwell::builder::Builder<'ctx>,
    width: u32,
    name: &str,
) -> PointerValue<'ctx> {
    let ty = ctx.i64_type().array_type(pk_words(width));
    crate::types::entry_alloca_in(b, ty, name)
}

// ---------------------------------------------------------------------------
// Runtime calls

fn wide_fn<'ctx>(cx: &LowerCtx<'ctx, '_, '_>) -> FunctionValue<'ctx> {
    let name = wide_rt::SV_RT_WIDE_SYMBOL;
    cx.module.get_function(name).unwrap_or_else(|| {
        let ptr = cx.ctx.ptr_type(AddressSpace::default());
        let i32t = cx.ctx.i32_type();
        let i64t = cx.ctx.i64_type();
        let ty = i64t.fn_type(
            &[
                i32t.into(),
                ptr.into(),
                ptr.into(),
                ptr.into(),
                i64t.into(),
                i64t.into(),
                i64t.into(),
                i64t.into(),
            ],
            false,
        );
        cx.module.add_function(name, ty, Some(Linkage::External))
    })
}

fn null<'ctx>(cx: &LowerCtx<'ctx, '_, '_>) -> PointerValue<'ctx> {
    cx.ctx.ptr_type(AddressSpace::default()).const_null()
}

fn c64<'ctx>(cx: &LowerCtx<'ctx, '_, '_>, v: u64) -> IntValue<'ctx> {
    cx.ctx.i64_type().const_int(v, false)
}

/// One `sv_rt_wide` call with unused pointers null. Returns the `i64` result
/// (the compare/reduce/truth ops return `val | unk << 1`; the rest 0).
fn emit<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    op: WideOp,
    dst: Option<PointerValue<'ctx>>,
    a: Option<PointerValue<'ctx>>,
    b: Option<PointerValue<'ctx>>,
    p: [IntValue<'ctx>; 4],
) -> IntValue<'ctx> {
    let n = || null(cx);
    cx.b.build_call(
        wide_fn(cx),
        &[
            cx.ctx.i32_type().const_int(op as u64, false).into(),
            dst.unwrap_or_else(n).into(),
            a.unwrap_or_else(n).into(),
            b.unwrap_or_else(n).into(),
            p[0].into(),
            p[1].into(),
            p[2].into(),
            p[3].into(),
        ],
        "sv_rt_wide_call",
    )
    .unwrap()
    .try_as_basic_value()
    .unwrap_basic()
    .into_int_value()
}

fn consts<'ctx>(cx: &LowerCtx<'ctx, '_, '_>, p: [u64; 4]) -> [IntValue<'ctx>; 4] {
    p.map(|v| c64(cx, v))
}

/// [`emit`] with constant parameters.
fn call<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    op: WideOp,
    dst: Option<PointerValue<'ctx>>,
    a: Option<PointerValue<'ctx>>,
    b: Option<PointerValue<'ctx>>,
    p: [u64; 4],
) -> IntValue<'ctx> {
    emit(cx, op, dst, a, b, consts(cx, p))
}

/// `sv_rt_wide` with every pointer given (the accessor conversions).
pub(crate) fn call_wide<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    op: WideOp,
    dst: PointerValue<'ctx>,
    a: PointerValue<'ctx>,
    b: PointerValue<'ctx>,
    p: [u64; 4],
) -> IntValue<'ctx> {
    call(cx, op, Some(dst), Some(a), Some(b), p)
}

// ---------------------------------------------------------------------------
// Narrow <-> memory

fn require_four_state(cx: &LowerCtx<'_, '_, '_>) -> Result<(), LoweringError> {
    if cx.repr != ValueRepr::FourState {
        return loud(
            cx,
            "a vector wider than 65535 bits needs the four-state value representation",
        );
    }
    Ok(())
}

/// A narrow value as a temporary image: its `i(2w)` zero-extended to whole
/// words (the same bit layout as `PK`).
fn spill<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    pv: PackedValue<'ctx>,
) -> Result<HugeValue<'ctx>, LoweringError> {
    require_four_state(cx)?;
    let words = pk_words(pv.width);
    let ptr = acquire(cx, words);
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let wide =
        cx.b.build_int_z_extend_or_bit_cast(pv.llvm, op.i_n(words * 64), "huge_spill")
            .unwrap();
    cx.b.build_store(ptr, wide)
        .unwrap()
        .set_alignment(8)
        .expect("alignment 8 is valid");
    Ok(HugeValue {
        ptr,
        width: pv.width,
        signed: pv.signed,
    })
}

/// The inverse of [`spill`]: a narrow image back into a [`PackedValue`].
fn load_narrow<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    ptr: PointerValue<'ctx>,
    width: u32,
    signed: bool,
) -> PackedValue<'ctx> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let words = pk_words(width);
    let v =
        cx.b.build_load(op.i_n(words * 64), ptr, "huge_load")
            .unwrap();
    inkwell::values::BasicValue::as_instruction_value(&v)
        .expect("a load is an instruction")
        .set_alignment(8)
        .expect("alignment 8 is valid");
    let v =
        cx.b.build_int_truncate_or_bit_cast(v.into_int_value(), op.i_n(2 * width), "huge_narrow")
            .unwrap();
    PackedValue {
        llvm: v,
        width,
        repr: ValueRepr::FourState,
        signed,
    }
}

/// A result image of `width` bits at `dst`: a [`HugeValue`] when huge, else
/// loaded into a [`PackedValue`] and the temporary released.
fn finish<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    dst: PointerValue<'ctx>,
    width: u32,
    signed: bool,
) -> LoweredValue<'ctx> {
    if is_huge_width(width) {
        return LoweredValue::Huge(HugeValue {
            ptr: dst,
            width,
            signed,
        });
    }
    let v = load_narrow(cx, dst, width, signed);
    release(dst);
    LoweredValue::Packed(v)
}

/// `lv` as an image: a huge value as is, a narrow one spilled.
fn to_huge<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    lv: LoweredValue<'ctx>,
    what: &str,
) -> Result<HugeValue<'ctx>, LoweringError> {
    match lv {
        LoweredValue::Huge(h) => Ok(h),
        LoweredValue::Packed(p) => spill(cx, p),
        LoweredValue::Real(_) | LoweredValue::Wide(_) | LoweredValue::Str(_) => loud(cx, what),
    }
}

fn lower_operand<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<HugeValue<'ctx>, LoweringError> {
    let lv = lower_expr_h(cx, e)?;
    to_huge(
        cx,
        lv,
        "an operand of a vector wider than the huge threshold must be an integral value",
    )
}

/// The `lower_expr` bridge: a huge value that still fits an LLVM integer
/// (`width <= WIDE_THRESHOLD`) becomes the [`PackedValue`] every consumer
/// without a memory-image path understands (one load; the image's bytes are
/// the packed `i(2N)`'s). A wider one stays an image.
pub(crate) fn bridge<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    v: LoweredValue<'ctx>,
) -> LoweredValue<'ctx> {
    match v {
        LoweredValue::Huge(h) if h.width <= WIDE_THRESHOLD => {
            let p = load_narrow(cx, h.ptr, h.width, h.signed);
            release(h.ptr);
            LoweredValue::Packed(p)
        }
        other => other,
    }
}

/// The one-bit four-state value of a compare/reduce/truth return (`val |
/// unk << 1`).
fn bit_result<'ctx>(cx: &LowerCtx<'ctx, '_, '_>, ret: IntValue<'ctx>) -> PackedValue<'ctx> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let two = cx.b.build_int_truncate(ret, op.i_n(2), "huge_bit").unwrap();
    PackedValue {
        llvm: two,
        width: 1,
        repr: ValueRepr::FourState,
        signed: false,
    }
}

fn type_width(cx: &LowerCtx<'_, '_, '_>, e: sv_lang::Expression<'_>) -> Result<u32, LoweringError> {
    e.expr_type()
        .map(|t| t.value_width())
        .ok_or_else(|| LoweringError::unsupported_at("expression without a type", &cx.fn_sym))
}

fn type_signed(e: sv_lang::Expression<'_>) -> bool {
    e.expr_type().is_some_and(|t| t.is_signed())
}

// ---------------------------------------------------------------------------
// Constants

fn bit_kind(b: &Bit) -> u64 {
    match b {
        Bit::Zero => wide_rt::FILL_ZERO,
        Bit::One => wide_rt::FILL_ONE,
        Bit::X => wide_rt::FILL_X,
        Bit::Z => wide_rt::FILL_Z,
    }
}

/// The bits (LSB first) of a constant expression, if it has them.
fn constant_bits(e: sv_lang::Expression<'_>) -> Option<Vec<Bit>> {
    let kind = e.kind();
    let lit = if kind == ExpressionKind::IntegerLiteral {
        e.integer_literal_value()
    } else if kind == ExpressionKind::StringLiteral {
        e.string_literal_int_value()
    } else if kind == ExpressionKind::UnbasedUnsizedIntegerLiteral {
        e.unbased_unsized_literal_value()
    } else if kind == ExpressionKind::Call && e.is_system_call() == Some(false) {
        // A user function call runs (its `$display`s and writes are effects a
        // folded value would drop; a `$swrite` into the result folds to X).
        return None;
    } else {
        // Any other constant expression: the evaluator's bits.
        return vj_sim::state::constant_expr_bits(&e);
    };
    let cv = lit?;
    let iv = cv.as_integer()?;
    vj_sim::state::svint_bits(iv).map(std::borrow::Cow::into_owned)
}

/// A `[words x i64]` private constant holding `words`, one per distinct image.
fn const_global<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    words: &[u64],
    width: u32,
) -> PointerValue<'ctx> {
    let mut h1 = std::collections::hash_map::DefaultHasher::new();
    let mut h2 = std::collections::hash_map::DefaultHasher::new();
    h1.write_u8(1);
    h2.write_u8(2);
    words.hash(&mut h1);
    words.hash(&mut h2);
    let name = format!(
        "huge_const_{width}_{:016x}{:016x}",
        h1.finish(),
        h2.finish()
    );
    if let Some(g) = cx.module.get_global(&name) {
        return g.as_pointer_value();
    }
    let i64t = cx.ctx.i64_type();
    let vals: Vec<IntValue<'ctx>> = words.iter().map(|w| i64t.const_int(*w, false)).collect();
    let ty = i64t.array_type(words.len() as u32);
    let g = cx.module.add_global(ty, None, &name);
    g.set_initializer(&i64t.const_array(&vals));
    g.set_constant(true);
    g.set_linkage(Linkage::Private);
    g.set_alignment(8);
    g.as_pointer_value()
}

/// A constant of huge type as an image: a `Fill` when every bit is alike, else
/// a private global.
fn constant<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<HugeValue<'ctx>>, LoweringError> {
    let Some(bits) = constant_bits(e) else {
        return Ok(None);
    };
    let width = type_width(cx, e)?;
    if bits.len() as u64 != u64::from(width) {
        return loud(
            cx,
            &format!("a constant of {} bits for a {width}-bit vector", bits.len()),
        );
    }
    let signed = type_signed(e);
    let kind = bit_kind(&bits[0]);
    if bits.iter().all(|b| bit_kind(b) == kind) {
        let dst = temp(cx, width);
        call(
            cx,
            WideOp::Fill,
            Some(dst),
            None,
            None,
            [u64::from(width), kind, 0, 0],
        );
        return Ok(Some(HugeValue {
            ptr: dst,
            width,
            signed,
        }));
    }
    let words = crate::value::packed_words(&bits, width, ValueRepr::FourState);
    Ok(Some(HugeValue {
        ptr: const_global(cx, &words, width),
        width,
        signed,
    }))
}

/// The constant hook of `expr::lower_constant`: a huge constant as a value.
pub(crate) fn lower_constant<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<LoweredValue<'ctx>>, LoweringError> {
    if !e.expr_type().is_some_and(|t| is_huge_type(&t)) {
        return Ok(None);
    }
    require_four_state(cx)?;
    Ok(constant(cx, e)?.map(LoweredValue::Huge))
}

// ---------------------------------------------------------------------------
// Which expressions are this module's

pub(crate) fn is_huge_expr(e: sv_lang::Expression<'_>) -> bool {
    e.expr_type().is_some_and(|t| is_huge_type(&t))
}

/// Whether an `inside` / `case` item candidate `c` (a value or a `[lo:hi]`
/// range) involves a huge vector.
pub(crate) fn candidate_is_huge(c: sv_lang::Expression<'_>) -> bool {
    if c.kind() == ExpressionKind::ValueRange {
        c.left().is_some_and(is_huge_expr) || c.right().is_some_and(is_huge_expr)
    } else {
        is_huge_expr(c)
    }
}

/// The candidates of an `inside` or `dist` expression `e`.
fn inside_parts<'d>(
    e: sv_lang::Expression<'d>,
) -> (
    Option<sv_lang::Expression<'d>>,
    Vec<sv_lang::Expression<'d>>,
) {
    if e.kind() == ExpressionKind::Dist {
        (
            e.dist_left(),
            e.dist_items().iter().map(|item| item.value()).collect(),
        )
    } else {
        (e.inside_left(), e.inside_range_list())
    }
}

/// The base of a select/member chain level, while that base is itself an
/// integral (non-floating) value this module can window into.
fn chain_base<'d>(e: sv_lang::Expression<'d>) -> Option<sv_lang::Expression<'d>> {
    let kind = e.kind();
    let base = if kind == ExpressionKind::ElementSelect || kind == ExpressionKind::RangeSelect {
        e.select_value()?
    } else if kind == ExpressionKind::MemberAccess {
        e.operand()?
    } else {
        // Not a chain level: `e` is the root.
        return None;
    };
    let ty = base.expr_type()?;
    (ty.is_integral() && !ty.is_floating()).then_some(base)
}

/// The expression a select/member chain bottoms out in (`e` itself when `e`
/// is not a chain).
fn chain_root<'d>(mut e: sv_lang::Expression<'d>) -> sv_lang::Expression<'d> {
    while let Some(base) = chain_base(e) {
        e = base;
    }
    e
}

/// Whether the lvalue/rvalue `e` is a select chain (or a whole variable)
/// rooted in a huge value.
pub(crate) fn rooted_in_huge(e: sv_lang::Expression<'_>) -> bool {
    is_huge_expr(chain_root(e))
}

/// Whether the lvalue `e` is a select chain (or a whole variable) rooted in a
/// huge variable of this process or function (an image in `cx.locals`): the
/// writes this module does natively.
pub(crate) fn huge_local_root(cx: &LowerCtx<'_, '_, '_>, e: sv_lang::Expression<'_>) -> bool {
    let root = chain_root(e);
    vj_sim::symid::is_value_name(root.kind())
        && root
            .referenced_symbol()
            .is_some_and(|s| cx.locals.get(&s.uid()).is_some_and(|l| is_huge_width(l.1)))
}

/// Whether the lvalue/rvalue `e` is a select chain (or a whole property)
/// rooted in a huge class property: its bits live in the object's field words,
/// which are exactly the `PK` image of the property.
pub(crate) fn class_prop_root(e: sv_lang::Expression<'_>) -> bool {
    let root = chain_root(e);
    is_huge_expr(root) && crate::classes::is_class_field_lvalue(root)
}

/// The whole huge class property `e` read into a fresh image.
fn read_class_prop<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let Some((h, slot, ty, dead)) = crate::classes::class_field_loc(cx, e)? else {
        return loud(cx, "internal: a class property expression with no location");
    };
    let width = ty.value_width();
    let dst = temp(cx, width);
    let live = crate::classes::field_get_into(cx, h, slot, dst);
    if let Some((name, at)) = dead {
        crate::classes::report_dead_handle_access(cx, live, at, &name, "read")?;
    }
    Ok(LoweredValue::Huge(HugeValue {
        ptr: dst,
        width,
        signed: ty.is_signed(),
    }))
}

/// Stores `v` through the select chain `lhs` into the huge class property it
/// is rooted in: the property's image is read, patched (a read-modify-write
/// for a partial write) and stored back.
fn write_class_prop<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    lhs: sv_lang::Expression<'d>,
    v: HugeValue<'ctx>,
) -> Result<(), LoweringError> {
    let (root, levels) = resolve(cx, lhs)?;
    let Some((h, slot, ty, dead)) = crate::classes::class_field_loc(cx, root)? else {
        return loud(cx, "internal: a class property lvalue with no location");
    };
    let width = ty.value_width();
    let cur = temp(cx, width);
    match levels.last() {
        None => {
            call(
                cx,
                WideOp::Resize,
                Some(cur),
                Some(v.ptr),
                None,
                [u64::from(width), v.info(), 0, 0],
            );
        }
        Some(last) => {
            if last.width != v.width {
                return loud(
                    cx,
                    "internal: the value assigned to a select of a huge vector is not as wide \
                     as the select",
                );
            }
            crate::classes::field_get_into(cx, h, slot, cur);
            write_levels(cx, cur, width, &levels, v)?;
        }
    }
    let live = crate::classes::field_set_from(cx, h, slot, cur);
    release(cur);
    if let Some((name, at)) = dead {
        crate::classes::report_dead_handle_access(cx, live, at, &name, "write")?;
    }
    Ok(())
}

/// Whether the lvalue/rvalue `e` is a select chain (or a whole element)
/// rooted in one element of a module memory whose elements are too wide for an
/// LLVM integer (`logic [70000:0] m [4]`, LRM 1800-2023 6.9.1): the row is read
/// into / written back from an image.
pub(crate) fn mem_elem_root(cx: &LowerCtx<'_, '_, '_>, e: sv_lang::Expression<'_>) -> bool {
    let root = chain_root(e);
    if root.kind() != ExpressionKind::ElementSelect
        || !root.expr_type().is_some_and(|t| is_image_type(&t))
    {
        return false;
    }
    let mut base = root.select_value();
    while let Some(b) = base
        && b.kind() == ExpressionKind::ElementSelect
    {
        base = b.select_value();
    }
    base.filter(|b| vj_sim::symid::is_value_name(b.kind()))
        .and_then(|b| b.referenced_symbol())
        .is_some_and(|s| cx.mem_signals.contains_key(&s.uid()))
}

/// The memory row `root` (`m[i]`) resolved to its slot: the row index, the
/// validity of the index, the element width and the accessor.
fn mem_row<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    root: sv_lang::Expression<'d>,
) -> Result<(crate::arrays::ArrayRef<'ctx>, crate::expr::MemorySignalInfo), LoweringError> {
    let Some((r, shape)) = crate::arrays::resolve_array_ref(cx, root)? else {
        return loud(cx, "internal: a memory element expression with no storage");
    };
    match r.backing {
        crate::arrays::Backing::Memory(info) if shape.dims.is_empty() => Ok((r, info)),
        _ => loud(
            cx,
            "an unpacked array whose elements are wider than 65535 bits, other than as a \
             module-level memory indexed down to one element",
        ),
    }
}

/// The element `e` (`m[i]`) of a memory of huge elements read into a fresh
/// image: all X (all 0 for a 2-state element) when the index is out of range
/// or has unknown bits (LRM 7.4.6).
fn read_mem_elem<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let (r, info) = mem_row(cx, e)?;
    let Some(acc) = cx.mem_accessor else {
        return loud(cx, "memory read with no accessor context");
    };
    let ty = e.expr_type().expect("a typed element");
    let width = info.elem_width;
    let dst = temp(cx, width);
    let fetch = |cx: &LowerCtx<'ctx, 'd, '_>| {
        crate::accessor::build_get_mem_huge(cx, acc.fns, acc.handle, info.slot, r.base, width, dst);
    };
    if r.valid.get_zero_extended_constant() == Some(1) {
        fetch(cx);
    } else {
        let func =
            cx.b.get_insert_block()
                .and_then(|b| b.get_parent())
                .expect("inside a function");
        let ok_bb = cx.ctx.append_basic_block(func, "hmem_ok");
        let bad_bb = cx.ctx.append_basic_block(func, "hmem_bad");
        let done_bb = cx.ctx.append_basic_block(func, "hmem_done");
        cx.b.build_conditional_branch(r.valid, ok_bb, bad_bb)
            .unwrap();
        cx.b.position_at_end(ok_bb);
        fetch(cx);
        cx.b.build_unconditional_branch(done_bb).unwrap();
        cx.b.position_at_end(bad_bb);
        init_image(cx, dst, width, ty.is_four_state())?;
        cx.b.build_unconditional_branch(done_bb).unwrap();
        cx.b.position_at_end(done_bb);
    }
    Ok(LoweredValue::Huge(HugeValue {
        ptr: dst,
        width,
        signed: ty.is_signed(),
    }))
}

/// Stores `v` through the select chain `lhs` into the memory element it is
/// rooted in: the row is read, patched (a read-modify-write for a partial
/// write) and stored back; an invalid index writes nothing.
fn write_mem_elem<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    lhs: sv_lang::Expression<'d>,
    v: HugeValue<'ctx>,
) -> Result<(), LoweringError> {
    let (root, levels) = resolve(cx, lhs)?;
    let (r, info) = mem_row(cx, root)?;
    let Some(acc) = cx.mem_accessor else {
        return loud(cx, "memory write with no accessor context");
    };
    let width = info.elem_width;
    if let Some(last) = levels.last()
        && last.width != v.width
    {
        return loud(
            cx,
            "internal: the value assigned to a select of a huge vector is not as wide as the \
             select",
        );
    }
    crate::arrays::guarded(cx, r.valid, || {
        let cur = temp(cx, width);
        if levels.is_empty() {
            call(
                cx,
                WideOp::Resize,
                Some(cur),
                Some(v.ptr),
                None,
                [u64::from(width), v.info(), 0, 0],
            );
        } else {
            crate::accessor::build_get_mem_huge(
                cx, acc.fns, acc.handle, info.slot, r.base, width, cur,
            );
            write_levels(cx, cur, width, &levels, v)?;
        }
        crate::accessor::build_set_mem_huge(cx, acc.fns, acc.handle, info.slot, r.base, width, cur);
        release(cur);
        Ok(())
    })
}

/// The module memory behind the row run `r` and the accessor to reach it, for
/// a whole-array operation on a memory whose elements are images (LRM
/// 1800-2023 6.9.1). A local array of such elements has no row storage.
fn mem_run<'ctx, 'm>(
    cx: &LowerCtx<'ctx, '_, 'm>,
    r: &crate::arrays::ArrayRef<'ctx>,
) -> Result<
    (
        crate::expr::MemorySignalInfo,
        crate::expr::MemAccessorCtx<'ctx, 'm>,
    ),
    LoweringError,
> {
    let crate::arrays::Backing::Memory(info) = r.backing else {
        return loud(
            cx,
            "a local unpacked array whose elements are wider than 65535 bits (LRM 1800-2023 \
             6.9.1); only module-level memories of such elements are supported",
        );
    };
    let Some(acc) = cx.mem_accessor else {
        return loud(cx, "memory access with no accessor context");
    };
    Ok((info, acc))
}

/// Row `k` (counted from the first row of the run `r`) of a memory of image
/// elements, read into a fresh image: all X (all 0 for a 2-state element) when
/// the run is invalid (an index that selected it was out of range or had
/// unknown bits, LRM 7.4.6).
pub(crate) fn load_mem_row<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    r: &crate::arrays::ArrayRef<'ctx>,
    k: IntValue<'ctx>,
    four_state: bool,
) -> Result<HugeValue<'ctx>, LoweringError> {
    let (info, acc) = mem_run(cx, r)?;
    let width = info.elem_width;
    let dst = temp(cx, width);
    let row = cx.b.build_int_add(r.base, k, "hmem_row").unwrap();
    let fetch = |cx: &LowerCtx<'ctx, '_, '_>| {
        crate::accessor::build_get_mem_huge(cx, acc.fns, acc.handle, info.slot, row, width, dst);
    };
    if r.valid.get_zero_extended_constant() == Some(1) {
        fetch(cx);
    } else {
        let func =
            cx.b.get_insert_block()
                .and_then(|b| b.get_parent())
                .expect("inside a function");
        let ok_bb = cx.ctx.append_basic_block(func, "hmem_ok");
        let bad_bb = cx.ctx.append_basic_block(func, "hmem_bad");
        let done_bb = cx.ctx.append_basic_block(func, "hmem_done");
        cx.b.build_conditional_branch(r.valid, ok_bb, bad_bb)
            .unwrap();
        cx.b.position_at_end(ok_bb);
        fetch(cx);
        cx.b.build_unconditional_branch(done_bb).unwrap();
        cx.b.position_at_end(bad_bb);
        init_image(cx, dst, width, four_state)?;
        cx.b.build_unconditional_branch(done_bb).unwrap();
        cx.b.position_at_end(done_bb);
    }
    Ok(HugeValue {
        ptr: dst,
        width,
        signed: false,
    })
}

/// Stores the image `v` to row `k` of the run `r` of a memory of image
/// elements, resized to the element width; nothing when the run is invalid.
pub(crate) fn store_mem_row<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    r: &crate::arrays::ArrayRef<'ctx>,
    k: IntValue<'ctx>,
    v: HugeValue<'ctx>,
) -> Result<(), LoweringError> {
    let (info, acc) = mem_run(cx, r)?;
    let width = info.elem_width;
    let row = cx.b.build_int_add(r.base, k, "hmem_row").unwrap();
    crate::arrays::guarded(cx, r.valid, || {
        let cur = if v.width == width {
            v.ptr
        } else {
            let cur = temp(cx, width);
            call(
                cx,
                WideOp::Resize,
                Some(cur),
                Some(v.ptr),
                None,
                [u64::from(width), v.info(), 0, 0],
            );
            cur
        };
        crate::accessor::build_set_mem_huge(cx, acc.fns, acc.handle, info.slot, row, width, cur);
        if cur != v.ptr {
            release(cur);
        }
        Ok(())
    })
}

/// The array element expression `e` evaluated into an image (an array
/// assignment pattern's element, LRM 1800-2023 10.9.1).
pub(crate) fn element_image<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<HugeValue<'ctx>, LoweringError> {
    require_four_state(cx)?;
    lower_operand(cx, e)
}

/// Whether `lower_expr` must route `e` through this module.
fn involves(e: sv_lang::Expression<'_>) -> bool {
    let sub = |x: Option<sv_lang::Expression<'_>>| x.is_some_and(is_huge_expr);
    let kind = e.kind();
    if kind == ExpressionKind::UnaryOp {
        is_huge_expr(e) || sub(e.operand())
    } else if kind == ExpressionKind::BinaryOp {
        // `&&`/`||`/`->`/`<->` read their operands' truth (`truth_packed`).
        !matches!(
            e.binary_op(),
            Some(
                BinaryOp::LogicalAnd
                    | BinaryOp::LogicalOr
                    | BinaryOp::LogicalImplication
                    | BinaryOp::LogicalEquivalence
            )
        ) && (is_huge_expr(e) || sub(e.left()) || sub(e.right()))
    } else if kind == ExpressionKind::Conversion {
        is_huge_expr(e) || sub(e.conversion_operand())
    } else if matches!(kind, ExpressionKind::Inside | ExpressionKind::Dist) {
        let (left, cands) = inside_parts(e);
        sub(left) || cands.into_iter().any(candidate_is_huge)
    } else if matches!(
        kind,
        ExpressionKind::ElementSelect | ExpressionKind::RangeSelect | ExpressionKind::MemberAccess
    ) {
        rooted_in_huge(e)
    } else if kind == ExpressionKind::Assignment {
        is_huge_expr(e) || sub(e.left()) || e.left().is_some_and(rooted_in_huge)
    } else {
        // Every other kind is this module's exactly when its own type is huge
        // (`lower` then names the ones it cannot lower).
        is_huge_expr(e)
    }
}

/// The `lower_expr` hook: `Some(value)` when `e` involves a huge vector.
pub(crate) fn try_lower<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<LoweredValue<'ctx>>, LoweringError> {
    if !involves(e) {
        return Ok(None);
    }
    require_four_state(cx)?;
    lower(cx, e).map(Some)
}

fn lower<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    if is_huge_expr(e)
        && let Some(c) = constant(cx, e)?
    {
        return Ok(LoweredValue::Huge(c));
    }
    // A select of a huge constant that is itself a small constant (`Cfg.XLEN`
    // of a struct parameter): read the folded value, not the whole root.
    if !is_huge_expr(e)
        && matches!(
            e.kind(),
            ExpressionKind::ElementSelect
                | ExpressionKind::RangeSelect
                | ExpressionKind::MemberAccess
        )
        && let Some(cv) = e.constant_value()
        && let Some(iv) = cv.as_integer()
        && !iv.has_unknown()
    {
        return crate::expr::lower_constant(cx, iv, Some(e));
    }
    if !native(cx, e)? {
        return fall_back(cx, e);
    }
    match e.kind() {
        ExpressionKind::NamedValue | ExpressionKind::HierarchicalValue => named(cx, e),
        ExpressionKind::UnaryOp => unary(cx, e),
        ExpressionKind::BinaryOp => binary(cx, e),
        ExpressionKind::ConditionalOp => ternary(cx, e),
        ExpressionKind::Concatenation => concatenation(cx, e),
        ExpressionKind::Replication => replication(cx, e),
        ExpressionKind::Conversion => conversion(cx, e),
        ExpressionKind::ElementSelect
        | ExpressionKind::RangeSelect
        | ExpressionKind::MemberAccess => read_chain(cx, e),
        ExpressionKind::Assignment => assignment(cx, e),
        ExpressionKind::LValueReference => lvalue_reference(cx),
        ExpressionKind::Inside | ExpressionKind::Dist => inside(cx, e),
        ExpressionKind::Call => {
            let name = e
                .system_subroutine()
                .map(|s| s.known_name_id())
                .ok_or_else(|| {
                    LoweringError::unsupported_at("a call without a name", &cx.fn_sym)
                })?;
            crate::sysfunc::lower_bit_sysfunc(cx, name, e)
                .unwrap_or_else(|| loud(cx, "internal: not a bit-level system function"))
        }
        other => Err(LoweringError::unsupported_at(
            &format!("a {other:?} expression involving a huge vector"),
            &cx.fn_sym,
        )),
    }
}

/// Whether this module has a native (memory-image) form of `e`; when not,
/// [`fall_back`] lowers it the ordinary way.
fn native<'d>(
    cx: &LowerCtx<'_, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<bool, LoweringError> {
    // Every other kind (a literal that is not constant-folded, a call of a
    // user function, ...) has no native form: `fall_back` lowers it the
    // ordinary way. The match below is therefore exhaustive over these.
    if !matches!(
        e.kind(),
        ExpressionKind::NamedValue
            | ExpressionKind::HierarchicalValue
            | ExpressionKind::Concatenation
            | ExpressionKind::Replication
            | ExpressionKind::ElementSelect
            | ExpressionKind::RangeSelect
            | ExpressionKind::MemberAccess
            | ExpressionKind::LValueReference
            | ExpressionKind::Inside
            | ExpressionKind::Dist
            | ExpressionKind::Call
            | ExpressionKind::Assignment
            | ExpressionKind::UnaryOp
            | ExpressionKind::BinaryOp
            | ExpressionKind::ConditionalOp
            | ExpressionKind::Conversion
    ) {
        return Ok(false);
    }
    Ok(match e.kind() {
        ExpressionKind::NamedValue
        | ExpressionKind::HierarchicalValue
        | ExpressionKind::Concatenation
        | ExpressionKind::Replication
        | ExpressionKind::ElementSelect
        | ExpressionKind::RangeSelect
        | ExpressionKind::MemberAccess
        | ExpressionKind::LValueReference
        | ExpressionKind::Inside
        | ExpressionKind::Dist => true,
        ExpressionKind::Call => is_bit_sysfunc(e),
        // A write is native into a huge variable of this process; into
        // anything else (an element of an unpacked array, a class property)
        // the ordinary store does it.
        ExpressionKind::Assignment => e
            .left()
            .is_some_and(|l| huge_local_root(cx, l) || class_prop_root(l) || mem_elem_root(cx, l)),
        // `++`/`--` write their operand: native into a huge variable of this
        // process, the ordinary read-modify-write otherwise.
        ExpressionKind::UnaryOp => match e.unary_op() {
            Some(
                UnaryOp::Preincrement
                | UnaryOp::Predecrement
                | UnaryOp::Postincrement
                | UnaryOp::Postdecrement,
            ) => e.operand().is_some_and(|o| {
                huge_local_root(cx, o) || class_prop_root(o) || mem_elem_root(cx, o)
            }),
            _ => true,
        },
        ExpressionKind::BinaryOp => true,
        ExpressionKind::ConditionalOp => {
            let conds = e.conditions();
            // Arms with side effects branch (`ternary_branching`); a pattern
            // condition is unsupported at every width.
            conds.len() == 1
                && conds[0].pattern().is_none()
                && e.true_value().is_some()
                && e.false_value().is_some()
        }
        ExpressionKind::Conversion => {
            use crate::expr::ConversionAction;
            let operand = e.conversion_operand();
            let to = e.expr_type();
            // A string converted to an integral value is by its bytes, with
            // no `Value` kind of its own to classify.
            let string_involved = to.as_ref().is_some_and(|t| t.is_string())
                || operand
                    .and_then(|o| o.expr_type())
                    .is_some_and(|t| t.is_string());
            match e
                .conversion_kind()
                .map(crate::expr::classify_conversion_kind)
            {
                Some(Ok(ConversionAction::Value)) => true,
                Some(Ok(ConversionAction::Stream)) => {
                    !string_involved
                        && to.is_some_and(|t| plain_stream_type(&t))
                        && operand.is_some_and(stream_native)
                }
                Some(Ok(ConversionAction::BitstreamCast)) => {
                    !string_involved
                        && to.is_some_and(|t| plain_stream_type(&t))
                        && operand.is_some_and(stream_native)
                }
                Some(Err(_)) | None => string_involved,
            }
        }
        other => unreachable!("{other:?} was filtered out above"),
    })
}

/// The ordinary lowering of an expression this module has no native form of:
/// its operands arrive as LLVM integers ([`crate::expr::lower_expr`] bridges
/// them), the operation runs on them, and a huge result is spilled back to an
/// image. Possible while everything involved fits `WIDE_THRESHOLD`; the
/// ordinary lowering's compile cost grows with the width, which is why the
/// operations people use on wide vectors have native forms.
fn fall_back<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let too_wide = |x: Option<sv_lang::Expression<'_>>| {
        x.and_then(|x| x.expr_type())
            .is_some_and(|t| t.is_integral() && t.value_width() > WIDE_THRESHOLD)
    };
    // A user subroutine call passes and returns images by pointer.
    if e.kind() == ExpressionKind::Call && e.is_system_call() == Some(false) {
        return fall_back_lowered(cx, e);
    }
    if too_wide(Some(e))
        || too_wide(e.operand())
        || too_wide(e.left())
        || too_wide(e.right())
        || too_wide(e.conversion_operand())
    {
        return loud(
            cx,
            &format!(
                "a {:?} expression on a vector wider than {WIDE_THRESHOLD} bits",
                e.kind()
            ),
        );
    }
    fall_back_lowered(cx, e)
}

fn fall_back_lowered<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    match lower_expr_regular(cx, e)? {
        LoweredValue::Packed(pv) if is_huge_expr(e) => Ok(LoweredValue::Huge(spill(cx, pv)?)),
        other => Ok(other),
    }
}

fn named<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let sym = e.referenced_symbol().ok_or_else(|| {
        LoweringError::unsupported_at("a name with no referenced symbol", &cx.fn_sym)
    })?;
    match cx.locals.get(&sym.uid()) {
        Some(&(ptr, width, signed)) if is_huge_width(width) => {
            Ok(LoweredValue::Huge(HugeValue { ptr, width, signed }))
        }
        Some(_) => loud(
            cx,
            "internal: a narrow local used where a vector wider than 65535 bits was expected",
        ),
        None if crate::classes::is_class_field_lvalue(e) => read_class_prop(cx, e),
        // Not a local of this process or function (a hierarchical
        // reference, ...): the ordinary read, while it fits.
        None => fall_back(cx, e),
    }
}

// ---------------------------------------------------------------------------
// Operators

fn unary<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let opd = e.operand().ok_or_else(|| {
        LoweringError::unsupported_at("unary operator without operand", &cx.fn_sym)
    })?;
    let uop = e
        .unary_op()
        .ok_or_else(|| LoweringError::unsupported_at("unary operator without kind", &cx.fn_sym))?;
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let reduce = |wop: WideOp| -> Result<PackedValue<'ctx>, LoweringError> {
        let v = lower_operand(cx, opd)?;
        let ret = call(cx, wop, None, Some(v.ptr), None, [0, v.info(), 0, 0]);
        release(v.ptr);
        Ok(bit_result(cx, ret))
    };
    let arith = |wop: WideOp| -> Result<LoweredValue<'ctx>, LoweringError> {
        let w = type_width(cx, e)?;
        let v = lower_operand(cx, opd)?;
        release(v.ptr);
        let dst = temp(cx, w);
        call(
            cx,
            wop,
            Some(dst),
            Some(v.ptr),
            None,
            [u64::from(w), v.info(), 0, 0],
        );
        Ok(finish(cx, dst, w, type_signed(e)))
    };
    match uop {
        // Unary plus changes neither the value nor its type.
        UnaryOp::Plus => lower_expr_h(cx, opd),
        UnaryOp::Minus => arith(WideOp::Neg),
        UnaryOp::BitwiseNot => arith(WideOp::Not),
        UnaryOp::BitwiseAnd => Ok(LoweredValue::Packed(reduce(WideOp::RedAnd)?)),
        UnaryOp::BitwiseOr => Ok(LoweredValue::Packed(reduce(WideOp::RedOr)?)),
        UnaryOp::BitwiseXor => Ok(LoweredValue::Packed(reduce(WideOp::RedXor)?)),
        UnaryOp::BitwiseNand => Ok(LoweredValue::Packed(op.not4(reduce(WideOp::RedAnd)?))),
        UnaryOp::BitwiseNor => Ok(LoweredValue::Packed(op.not4(reduce(WideOp::RedOr)?))),
        UnaryOp::BitwiseXnor => Ok(LoweredValue::Packed(op.not4(reduce(WideOp::RedXor)?))),
        UnaryOp::LogicalNot => Ok(LoweredValue::Packed(op.lnot4(reduce(WideOp::RedOr)?))),
        UnaryOp::Preincrement
        | UnaryOp::Predecrement
        | UnaryOp::Postincrement
        | UnaryOp::Postdecrement => increment(cx, e, opd, uop),
    }
}

/// `++x`, `--x`, `x++`, `x--` on a huge vector (or a select of one): the
/// operand read, one added or subtracted at the operand's own width (LRM
/// 11.4.2: the same as `x = x + 1`, the operation done once), the sum stored
/// back, and the new value (pre) or the old one (post) as the result.
fn increment<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    opd: sv_lang::Expression<'d>,
    uop: UnaryOp,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let w = type_width(cx, e)?;
    let signed = type_signed(e);
    let cur = lower_operand(cx, opd)?;
    let post = matches!(uop, UnaryOp::Postincrement | UnaryOp::Postdecrement);
    let wop = if matches!(uop, UnaryOp::Preincrement | UnaryOp::Postincrement) {
        WideOp::Add
    } else {
        WideOp::Sub
    };
    // A whole variable reads as its own shadow, which the store below
    // overwrites: the old value a postfix operator returns is copied first.
    let old = post.then(|| {
        let copy = temp(cx, w);
        call(
            cx,
            WideOp::Resize,
            Some(copy),
            Some(cur.ptr),
            None,
            [u64::from(w), cur.info(), 0, 0],
        );
        copy
    });
    let one = spill(
        cx,
        PackedValue {
            llvm: OpBuilder {
                ctx: cx.ctx,
                b: &cx.b,
            }
            .i_n(2)
            .const_int(1, false),
            width: 1,
            repr: ValueRepr::FourState,
            signed: false,
        },
    )?;
    let new = temp(cx, w);
    call(
        cx,
        wop,
        Some(new),
        Some(cur.ptr),
        Some(one.ptr),
        [u64::from(w), cur.info(), one.info(), u64::from(cur.signed)],
    );
    release(cur.ptr);
    release(one.ptr);
    let stored = HugeValue {
        ptr: new,
        width: w,
        signed,
    };
    if class_prop_root(opd) && !huge_local_root(cx, opd) {
        write_class_prop(cx, opd, stored)?;
    } else if mem_elem_root(cx, opd) {
        write_mem_elem(cx, opd, stored)?;
    } else {
        let (shadow, _) = write_chain(cx, opd, stored)?;
        write_through_commit(cx, shadow);
    }
    match old {
        Some(copy) => {
            release(new);
            Ok(finish(cx, copy, w, signed))
        }
        None => Ok(finish(cx, new, w, signed)),
    }
}

fn binary<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let (Some(l), Some(r), Some(bop)) = (e.left(), e.right(), e.binary_op()) else {
        return loud(cx, "binary operator without operands");
    };
    let arith = |wop: WideOp| -> Result<LoweredValue<'ctx>, LoweringError> {
        let w = type_width(cx, e)?;
        let lv = lower_operand(cx, l)?;
        let rv = lower_operand(cx, r)?;
        release(lv.ptr);
        release(rv.ptr);
        let dst = temp(cx, w);
        call(
            cx,
            wop,
            Some(dst),
            Some(lv.ptr),
            Some(rv.ptr),
            [u64::from(w), lv.info(), rv.info(), u64::from(lv.signed)],
        );
        Ok(finish(cx, dst, w, type_signed(e)))
    };
    // The shift amount is self-determined and unsigned: any width.
    let shift = |wop: WideOp| -> Result<LoweredValue<'ctx>, LoweringError> {
        let w = type_width(cx, e)?;
        let lv = lower_operand(cx, l)?;
        let rv = lower_operand(cx, r)?;
        release(lv.ptr);
        release(rv.ptr);
        let dst = temp(cx, w);
        call(
            cx,
            wop,
            Some(dst),
            Some(lv.ptr),
            Some(rv.ptr),
            [
                u64::from(w),
                lv.info(),
                rv.info(),
                u64::from(wop == WideOp::Ashr),
            ],
        );
        Ok(finish(cx, dst, w, type_signed(e)))
    };
    let compare = |wop: WideOp| -> Result<LoweredValue<'ctx>, LoweringError> {
        let lv = lower_operand(cx, l)?;
        let rv = lower_operand(cx, r)?;
        let ret = call(
            cx,
            wop,
            None,
            Some(lv.ptr),
            Some(rv.ptr),
            [
                u64::from(lv.width.max(rv.width)),
                lv.info(),
                rv.info(),
                u64::from(lv.signed),
            ],
        );
        release(lv.ptr);
        release(rv.ptr);
        Ok(LoweredValue::Packed(bit_result(cx, ret)))
    };
    match bop {
        BinaryOp::Add => arith(WideOp::Add),
        BinaryOp::Subtract => arith(WideOp::Sub),
        BinaryOp::Multiply => arith(WideOp::Mul),
        BinaryOp::Divide => arith(WideOp::Div),
        BinaryOp::Mod => arith(WideOp::Mod),
        BinaryOp::BinaryAnd => arith(WideOp::And),
        BinaryOp::BinaryOr => arith(WideOp::Or),
        BinaryOp::BinaryXor => arith(WideOp::Xor),
        BinaryOp::BinaryXnor => arith(WideOp::Xnor),
        BinaryOp::LogicalShiftLeft | BinaryOp::ArithmeticShiftLeft => shift(WideOp::Shl),
        BinaryOp::LogicalShiftRight => shift(WideOp::Shr),
        // `>>>` is arithmetic only when the shifted operand is signed.
        BinaryOp::ArithmeticShiftRight => shift(if type_signed(e) {
            WideOp::Ashr
        } else {
            WideOp::Shr
        }),
        BinaryOp::Equality => compare(WideOp::Eq),
        BinaryOp::Inequality => compare(WideOp::Ne),
        BinaryOp::CaseEquality => compare(WideOp::CaseEq),
        BinaryOp::CaseInequality => compare(WideOp::CaseNe),
        BinaryOp::WildcardEquality => compare(WideOp::WildEq),
        BinaryOp::WildcardInequality => compare(WideOp::WildNe),
        BinaryOp::LessThan => compare(WideOp::Lt),
        BinaryOp::LessThanEqual => compare(WideOp::Le),
        BinaryOp::GreaterThan => compare(WideOp::Gt),
        BinaryOp::GreaterThanEqual => compare(WideOp::Ge),
        // The exponent is self-determined and keeps its own signedness.
        BinaryOp::Power => {
            let w = type_width(cx, e)?;
            let lv = lower_operand(cx, l)?;
            let rv = lower_operand(cx, r)?;
            release(lv.ptr);
            release(rv.ptr);
            let dst = temp(cx, w);
            call(
                cx,
                WideOp::Pow,
                Some(dst),
                Some(lv.ptr),
                Some(rv.ptr),
                [u64::from(w), lv.info(), rv.info(), 0],
            );
            Ok(finish(cx, dst, w, type_signed(e)))
        }
        BinaryOp::LogicalAnd
        | BinaryOp::LogicalOr
        | BinaryOp::LogicalImplication
        | BinaryOp::LogicalEquivalence => loud(
            cx,
            "internal: a logical operator reached the huge-vector lowering",
        ),
    }
}

/// Which truth a huge condition is read for.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(crate) enum TruthMode {
    /// `if`/`while`/`&&`/`||`: true only when the whole value is known and
    /// nonzero (the narrow `is_definitely_true`).
    Strict,
    /// `?:`: true when any bit is a known 1 (`ternary_is_definitely_true`).
    Ternary,
}

/// `lv` as a [`PackedValue`] fit for the truth tests: a huge value becomes a
/// one-bit proxy (known 1 / known 0 / unknown) with the same verdicts, any
/// other value is `into_packed`.
pub(crate) fn truth_packed<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    lv: LoweredValue<'ctx>,
    mode: TruthMode,
    what: &str,
) -> Result<PackedValue<'ctx>, LoweringError> {
    match lv {
        LoweredValue::Huge(h) => {
            let wop = match mode {
                TruthMode::Strict => WideOp::Truth,
                TruthMode::Ternary => WideOp::RedOr,
            };
            let ret = call(cx, wop, None, Some(h.ptr), None, [0, h.info(), 0, 0]);
            release(h.ptr);
            Ok(bit_result(cx, ret))
        }
        LoweredValue::Real(rv) => Ok(crate::real::real_truth_packed(cx, rv)),
        other => other.into_packed(what, &cx.fn_sym),
    }
}

/// A selector or label of a `case` / `inside` over a vector wider than the
/// threshold, as an image (a narrow one spilled).
pub(crate) fn lower_selector<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<HugeValue<'ctx>, LoweringError> {
    require_four_state(cx)?;
    lower_operand(cx, e)
}

/// `l` and `r` compared by `wop` at the wider of their widths, signed only when
/// both are (LRM 11.8.1): the one-bit result, `l` and `r` left alive.
pub(crate) fn compare_images<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    wop: WideOp,
    l: HugeValue<'ctx>,
    r: HugeValue<'ctx>,
) -> PackedValue<'ctx> {
    let signed = l.signed && r.signed;
    let (l, r) = (HugeValue { signed, ..l }, HugeValue { signed, ..r });
    let ret = call(
        cx,
        wop,
        None,
        Some(l.ptr),
        Some(r.ptr),
        [
            u64::from(l.width.max(r.width)),
            l.info(),
            r.info(),
            u64::from(signed),
        ],
    );
    bit_result(cx, ret)
}

/// `left inside {[a +/- b]}` / `{[a +%- b]}` over images (LRM 1800-2023
/// 11.4.13): the range is `[a-t : a+t]` with `t = type(a)'(b)` (absolute) or
/// `type(a)'(a*b/100.0)` (relative, truncated toward zero), the bounds
/// swapped when `a-t > a+t`. Matching either orientation (`lo <= x <= hi` or
/// `hi <= x <= lo`) is the same predicate without the swap (the second only
/// fires for a swapped range or `lo == hi`). An unknown bit in `a` or `b`
/// makes both bounds X, so the match is X, like the narrow path.
fn tolerance_contribution<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    left: HugeValue<'ctx>,
    cand: sv_lang::Expression<'d>,
    relative: bool,
) -> Result<PackedValue<'ctx>, LoweringError> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let (Some(ae), Some(be)) = (cand.left(), cand.right()) else {
        return loud(cx, "an inside tolerance range missing an operand");
    };
    let a = lower_operand(cx, ae)?;
    let b = lower_operand(cx, be)?;
    let wa = a.width;
    // The tolerance image `t` and the operand info it is read with.
    let (t, t_info, t_tmp) = if relative {
        let w = a.width + b.width + 1;
        let prod = temp(cx, w);
        call(
            cx,
            WideOp::Mul,
            Some(prod),
            Some(a.ptr),
            Some(b.ptr),
            [u64::from(w), a.info(), b.info(), 0],
        );
        let hundred = spill(
            cx,
            op.pack(
                op.i_n(32).const_int(100, false),
                op.i_n(32).const_zero(),
                32,
                cx.repr,
                false,
            ),
        )?;
        let q = temp(cx, w);
        // Signed division truncates toward zero, like the real quotient.
        call(
            cx,
            WideOp::Div,
            Some(q),
            Some(prod),
            Some(hundred.ptr),
            [u64::from(w), wide_rt::info(w, true), hundred.info(), 1],
        );
        release(prod);
        release(hundred.ptr);
        (q, wide_rt::info(w, false), Some(q))
    } else {
        (b.ptr, b.info(), None)
    };
    let lo = temp(cx, wa);
    let hi = temp(cx, wa);
    for (wop, dst) in [(WideOp::Sub, lo), (WideOp::Add, hi)] {
        call(
            cx,
            wop,
            Some(dst),
            Some(a.ptr),
            Some(t),
            [u64::from(wa), a.info(), t_info, 0],
        );
    }
    release(a.ptr);
    release(b.ptr);
    if let Some(q) = t_tmp {
        release(q);
    }
    let lo = HugeValue {
        ptr: lo,
        width: wa,
        signed: a.signed,
    };
    let hi = HugeValue {
        ptr: hi,
        width: wa,
        signed: a.signed,
    };
    let fwd = op.and4(
        compare_images(cx, WideOp::Ge, left, lo),
        compare_images(cx, WideOp::Le, left, hi),
    );
    let rev = op.and4(
        compare_images(cx, WideOp::Ge, left, hi),
        compare_images(cx, WideOp::Le, left, lo),
    );
    release(lo.ptr);
    release(hi.ptr);
    Ok(op.or4(fwd, rev))
}

/// One `inside` candidate's three-valued contribution (LRM 11.4.13): `left
/// ==? value`, or `lo <= left && left <= hi` for a range.
fn inside_contribution<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    left: HugeValue<'ctx>,
    cand: sv_lang::Expression<'d>,
) -> Result<PackedValue<'ctx>, LoweringError> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    if cand.kind() == ExpressionKind::ValueRange {
        if let Some(relative) = crate::expr::tolerance_op(cand) {
            return tolerance_contribution(cx, left, cand, relative);
        }
        let (Some(lo), Some(hi)) = (cand.left(), cand.right()) else {
            return loud(cx, "an inside range missing a bound");
        };
        // `$` is the minimum / maximum of the left operand's type: that side
        // always holds, so the operand itself stands in for it (an X operand
        // stays X), like the narrow path.
        let ge = if crate::expr::is_open_bound(Some(lo)) {
            compare_images(cx, WideOp::Ge, left, left)
        } else {
            let lo = lower_operand(cx, lo)?;
            let ge = compare_images(cx, WideOp::Ge, left, lo);
            release(lo.ptr);
            ge
        };
        let le = if crate::expr::is_open_bound(Some(hi)) {
            compare_images(cx, WideOp::Le, left, left)
        } else {
            let hi = lower_operand(cx, hi)?;
            let le = compare_images(cx, WideOp::Le, left, hi);
            release(hi.ptr);
            le
        };
        return Ok(op.and4(ge, le));
    }
    let item = lower_operand(cx, cand)?;
    let m = compare_images(cx, WideOp::WildEq, left, item);
    release(item.ptr);
    Ok(m)
}

/// `x inside {...}` (and `x dist {...}`, which simulates as `inside`) with a
/// huge operand or candidate.
fn inside<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let (left, cands) = inside_parts(e);
    let Some(left) = left else {
        return loud(cx, "inside missing its left operand");
    };
    if cands.is_empty() {
        return loud(cx, "inside with an empty candidate list");
    }
    let left = lower_operand(cx, left)?;
    let mut acc: Option<PackedValue<'ctx>> = None;
    for cand in cands {
        let c = inside_contribution(cx, left, cand)?;
        acc = Some(match acc {
            None => c,
            Some(prev) => op.or4(prev, c),
        });
    }
    release(left.ptr);
    Ok(LoweredValue::Packed(
        acc.expect("candidates non-empty, checked above"),
    ))
}

/// Whether item label `label` of a `case` of kind `kind` matches the huge
/// selector `sel` (an `i1`; an ambiguous comparison does not match).
pub(crate) fn case_label_matches<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    kind: sv_lang::CaseStatementCondition,
    sel: HugeValue<'ctx>,
    label: sv_lang::Expression<'d>,
) -> Result<IntValue<'ctx>, LoweringError> {
    use sv_lang::CaseStatementCondition as K;
    let hit = match kind {
        K::Inside => inside_contribution(cx, sel, label)?,
        K::Normal => {
            let item = lower_operand(cx, label)?;
            let r = compare_images(cx, WideOp::CaseEq, sel, item);
            release(item.ptr);
            r
        }
        K::WildcardXOrZ | K::WildcardJustZ => {
            let item = lower_operand(cx, label)?;
            let signed = sel.signed && item.signed;
            let (l, r) = (HugeValue { signed, ..sel }, HugeValue { signed, ..item });
            let ret = call(
                cx,
                WideOp::CaseMatch,
                None,
                Some(l.ptr),
                Some(r.ptr),
                [
                    u64::from(l.width.max(r.width)),
                    l.info(),
                    r.info(),
                    u64::from(kind == K::WildcardJustZ),
                ],
            );
            release(item.ptr);
            bit_result(cx, ret)
        }
    };
    Ok(crate::ops::is_definitely_true(cx.ctx, &cx.b, hit))
}

/// Whether `e` is a call to one of the bit-level system functions
/// ([`crate::sysfunc::lower_bit_sysfunc`]).
fn is_bit_sysfunc(e: sv_lang::Expression<'_>) -> bool {
    e.kind() == ExpressionKind::Call
        && e.is_system_call() == Some(true)
        && e.system_subroutine()
            .is_some_and(|s| crate::sysfunc::is_bit_sysfunc_name(s.known_name_id()))
}

/// A bit-level system function (`$signed`, `$countones`, `$clog2`, ...)
/// applied to the huge value `h` (LRM 1800-2023 20.5, 20.9), computed in the
/// runtime on the image: the counts and the log never build an `iN`.
pub(crate) fn sysfunc<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    name: sv_lang::KnownSystemName,
    e: sv_lang::Expression<'_>,
    h: HugeValue<'ctx>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    use sv_lang::KnownSystemName as K;
    if matches!(name, K::Signed | K::Unsigned) {
        // The same bits, the other signedness.
        return Ok(LoweredValue::Huge(HugeValue {
            signed: name == K::Signed,
            ..h
        }));
    }
    let ret = e.expr_type().ok_or_else(|| {
        LoweringError::unsupported_at("system function call has no type", &cx.fn_sym)
    })?;
    let rw = ret.value_width();
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let count = |mask: u64| {
        call(
            cx,
            WideOp::Count,
            None,
            Some(h.ptr),
            None,
            [mask, h.info(), 0, 0],
        )
    };
    let i64t = cx.ctx.i64_type();
    let (value, unknown) = match name {
        K::CountOnes => (count(2), None),
        K::OneHot | K::OneHot0 => {
            let pred = if name == K::OneHot {
                IntPredicate::EQ
            } else {
                IntPredicate::ULE
            };
            let bit =
                cx.b.build_int_compare(pred, count(2), i64t.const_int(1, false), "huge_onehot")
                    .unwrap();
            (bit, None)
        }
        K::IsUnknown => {
            let n = count(12);
            let bit =
                cx.b.build_int_compare(IntPredicate::NE, n, i64t.const_zero(), "huge_isunk")
                    .unwrap();
            (bit, None)
        }
        K::CountBits => {
            // Each control bit selects a state (LRM 1800-2023 20.9): the
            // mask has bit 0 for 0, bit 1 for 1, bit 2 for X and bit 3 for Z. A
            // constant control folds; a run-time one picks its mask bit
            // from bit 0 of the argument (an unknown bit selects X when its
            // value lane is 0, Z when 1, as for narrow operands).
            let mut mask = i64t.const_zero();
            for c in e.operands().iter().skip(1) {
                let bit = c
                    .constant_value()
                    .as_ref()
                    .and_then(|cv| cv.as_integer())
                    .and_then(|i| i.bit(0));
                let lane = if let Some(bit) = bit {
                    i64t.const_int(
                        match bit {
                            Bit::Zero => 1,
                            Bit::One => 2,
                            Bit::X => 4,
                            Bit::Z => 8,
                        },
                        false,
                    )
                } else {
                    let cv = lower_expr_h(cx, *c)?.into_packed(
                        "a $countbits control bit of a real or wide value",
                        &cx.fn_sym,
                    )?;
                    let i1 = cx.ctx.bool_type();
                    let cval =
                        cx.b.build_int_truncate(op.lane_val(cv), i1, "hcb_v")
                            .unwrap();
                    let cunk =
                        cx.b.build_int_truncate(op.lane_unk(cv), i1, "hcb_u")
                            .unwrap();
                    let pick = |t: u64, f: u64, name: &str| {
                        cx.b.build_select(cval, c64(cx, t), c64(cx, f), name)
                            .unwrap()
                            .into_int_value()
                    };
                    cx.b.build_select(cunk, pick(8, 4, "hcb_x"), pick(2, 1, "hcb_k"), "hcb_lane")
                        .unwrap()
                        .into_int_value()
                };
                mask = cx.b.build_or(mask, lane, "hcb_mask").unwrap();
            }
            let n = emit(
                cx,
                WideOp::Count,
                None,
                Some(h.ptr),
                None,
                [mask, c64(cx, h.info()), c64(cx, 0), c64(cx, 0)],
            );
            (n, None)
        }
        K::Clog2 => {
            let r = call(
                cx,
                WideOp::Clog2,
                None,
                Some(h.ptr),
                None,
                [0, h.info(), 0, 0],
            );
            let any_unk =
                cx.b.build_int_compare(IntPredicate::EQ, r, i64t.const_all_ones(), "huge_clog2_x")
                    .unwrap();
            (r, Some(any_unk))
        }
        _ => unreachable!("filtered by is_bit_sysfunc"),
    };
    release(h.ptr);
    let rt = op.i_n(rw);
    let v = crate::sysfunc::fit(cx, value, rw);
    let (v, unk) = match unknown {
        Some(x) => (
            cx.b.build_select(x, rt.const_zero(), v, "huge_clog2_v")
                .unwrap()
                .into_int_value(),
            cx.b.build_select(x, rt.const_all_ones(), rt.const_zero(), "huge_clog2_u")
                .unwrap()
                .into_int_value(),
        ),
        None => (v, rt.const_zero()),
    };
    Ok(LoweredValue::Packed(op.pack(
        v,
        unk,
        rw,
        cx.repr,
        ret.is_signed(),
    )))
}

fn ternary<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let conds = e.conditions();
    if conds.len() != 1 || conds[0].pattern().is_some() {
        return loud(
            cx,
            "a pattern-matching or multi-condition `?:` with a vector wider than 65535 bits",
        );
    }
    let (Some(t), Some(f)) = (e.true_value(), e.false_value()) else {
        return loud(cx, "ternary without both arms");
    };
    let clean = |arm: sv_lang::Expression<'d>| {
        crate::expr::is_side_effect_free(&arm, &cx.fn_sym, &crate::coverage::NoopCoverageSink)
    };
    let w = type_width(cx, e)?;
    if !(clean(t)? && clean(f)?) {
        return ternary_branching(cx, e, conds[0].expr(), t, f, w);
    }
    let cond = truth_packed(
        cx,
        lower_expr_h(cx, conds[0].expr())?,
        TruthMode::Ternary,
        "ternary condition must be an integral (non-real) value",
    )?;
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let is_true = crate::ops::ternary_is_definitely_true(cx.ctx, &cx.b, cond);
    let is_false = crate::ops::is_definitely_false(cx.ctx, &cx.b, cond);
    let i64t = cx.ctx.i64_type();
    let mode =
        cx.b.build_select(
            is_true,
            i64t.const_int(1, false),
            cx.b.build_select(is_false, i64t.const_zero(), i64t.const_int(2, false), "m")
                .unwrap()
                .into_int_value(),
            "huge_tmode",
        )
        .unwrap()
        .into_int_value();
    let _ = op;
    let tv = lower_operand(cx, t)?;
    let fv = lower_operand(cx, f)?;
    release(tv.ptr);
    release(fv.ptr);
    let dst = temp(cx, w);
    emit(
        cx,
        WideOp::Ternary,
        Some(dst),
        Some(tv.ptr),
        Some(fv.ptr),
        [
            c64(cx, u64::from(w)),
            c64(cx, tv.info()),
            c64(cx, fv.info()),
            mode,
        ],
    );
    Ok(finish(cx, dst, w, type_signed(e)))
}

/// `c ? t : f` where an arm has side effects (LRM 11.4.11): only the chosen arm
/// runs when the condition is known, both run (in order) when it is X/Z and
/// the result keeps the bits they agree on. The result lives in one image
/// every path writes.
fn ternary_branching<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    cond_expr: sv_lang::Expression<'d>,
    t: sv_lang::Expression<'d>,
    f: sv_lang::Expression<'d>,
    w: u32,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let cond = truth_packed(
        cx,
        lower_expr_h(cx, cond_expr)?,
        TruthMode::Ternary,
        "ternary condition must be an integral (non-real) value",
    )?;
    let is_true = crate::ops::ternary_is_definitely_true(cx.ctx, &cx.b, cond);
    let is_false = crate::ops::is_definitely_false(cx.ctx, &cx.b, cond);
    let dst = temp(cx, w);
    let func =
        cx.b.get_insert_block()
            .and_then(|bb| bb.get_parent())
            .expect("lowering happens inside a function");
    let then_bb = cx.ctx.append_basic_block(func, "huge_tern_then");
    let check_bb = cx.ctx.append_basic_block(func, "huge_tern_check");
    let else_bb = cx.ctx.append_basic_block(func, "huge_tern_else");
    let ambig_bb = cx.ctx.append_basic_block(func, "huge_tern_ambig");
    let merge_bb = cx.ctx.append_basic_block(func, "huge_tern_merge");
    cx.b.build_conditional_branch(is_true, then_bb, check_bb)
        .unwrap();
    cx.b.position_at_end(check_bb);
    cx.b.build_conditional_branch(is_false, else_bb, ambig_bb)
        .unwrap();
    let chosen = |arm: sv_lang::Expression<'d>| -> Result<(), LoweringError> {
        let v = lower_operand(cx, arm)?;
        call(
            cx,
            WideOp::Resize,
            Some(dst),
            Some(v.ptr),
            None,
            [u64::from(w), v.info(), 0, 0],
        );
        release(v.ptr);
        cx.b.build_unconditional_branch(merge_bb).unwrap();
        Ok(())
    };
    cx.b.position_at_end(then_bb);
    chosen(t)?;
    cx.b.position_at_end(else_bb);
    chosen(f)?;
    cx.b.position_at_end(ambig_bb);
    let tv = lower_operand(cx, t)?;
    let fv = lower_operand(cx, f)?;
    call(
        cx,
        WideOp::Ternary,
        Some(dst),
        Some(tv.ptr),
        Some(fv.ptr),
        [u64::from(w), tv.info(), fv.info(), 2],
    );
    release(tv.ptr);
    release(fv.ptr);
    cx.b.build_unconditional_branch(merge_bb).unwrap();
    cx.b.position_at_end(merge_bb);
    Ok(finish(cx, dst, w, type_signed(e)))
}

fn concatenation<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let parts = e.operands();
    if parts.is_empty() {
        return loud(cx, "empty concatenation");
    }
    let w = type_width(cx, e)?;
    let dst = temp(cx, w);
    call(
        cx,
        WideOp::Fill,
        Some(dst),
        None,
        None,
        [u64::from(w), wide_rt::FILL_ZERO, 0, 0],
    );
    // Operands run MSB first; a zero-count replication adds no bits.
    let mut offset = u64::from(w);
    for part in parts {
        if crate::expr::is_zero_replication(&part) {
            // The dropped operand's own side effects still run once, in
            // source order (`{0{f()}}` calls `f`; expr.rs does the same).
            if let Some(inner) = part.replication_concat() {
                release(lower_operand(cx, inner)?.ptr);
            }
            continue;
        }
        let v = lower_operand(cx, part)?;
        let Some(next) = offset.checked_sub(u64::from(v.width)) else {
            return loud(cx, "concatenation operands wider than the concatenation");
        };
        offset = next;
        call(
            cx,
            WideOp::Insert,
            Some(dst),
            Some(v.ptr),
            None,
            [u64::from(v.width), u64::from(w), offset, 0],
        );
        release(v.ptr);
    }
    if offset != 0 {
        return loud(
            cx,
            "internal: concatenation operand widths do not sum to the concatenation's width",
        );
    }
    Ok(finish(cx, dst, w, type_signed(e)))
}

fn replication<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let count_expr = e
        .replication_count()
        .ok_or_else(|| LoweringError::unsupported_at("replication without a count", &cx.fn_sym))?;
    let count = require_const_i64(
        count_expr,
        "replication count must be a compile-time constant",
        &cx.fn_sym,
    )?;
    if count <= 0 {
        return loud(cx, "replication with a non-positive count");
    }
    let unit = e
        .replication_concat()
        .ok_or_else(|| LoweringError::unsupported_at("replication without a body", &cx.fn_sym))?;
    let w = type_width(cx, e)?;
    let v = lower_operand(cx, unit)?;
    if u64::from(v.width) * count as u64 != u64::from(w) {
        return loud(
            cx,
            "internal: replication width does not match count times the body",
        );
    }
    release(v.ptr);
    let dst = temp(cx, w);
    call(
        cx,
        WideOp::Repl,
        Some(dst),
        Some(v.ptr),
        None,
        [u64::from(w), v.info(), count as u64, 0],
    );
    Ok(finish(cx, dst, w, type_signed(e)))
}

/// Whether a value of type `t` streams as its own stored bits (LRM 11.4.14):
/// an integral value -- not a real, a string, a dynamically sized array or an
/// unpacked aggregate (whose element order differs from its storage's).
fn plain_stream_type(t: &sv_lang::Type<'_>) -> bool {
    t.is_integral()
        && !t.is_floating()
        && !t.is_unpacked_array()
        && !vj_sim::state::storage::is_unpacked_aggregate(t)
}

/// Whether the stream `inner` (a `Streaming` expression or a lone operand)
/// is made of plain operands only, with no `with [...]` range.
fn stream_native(inner: sv_lang::Expression<'_>) -> bool {
    if inner.kind() == ExpressionKind::Streaming {
        let streams = inner.streams();
        !streams.is_empty()
            && streams
                .iter()
                .all(|s| s.with_expr().is_none() && stream_native(s.operand()))
    } else {
        inner.expr_type().is_some_and(|t| plain_stream_type(&t))
    }
}

/// The bit-stream of `e` (a `Streaming` expression or a lone plain operand)
/// as an image: the operands' bits, first one most significant, cut into
/// `n`-bit slices from the least significant end and reversed for `{<<n{}}`.
fn stream_image<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<HugeValue<'ctx>, LoweringError> {
    if e.kind() != ExpressionKind::Streaming {
        return lower_operand(cx, e);
    }
    let mut parts = Vec::new();
    for s in e.streams() {
        parts.push(stream_image(cx, s.operand())?);
    }
    let total: u32 = parts.iter().map(|p| p.width).sum();
    let dst = temp(cx, total);
    call(
        cx,
        WideOp::Fill,
        Some(dst),
        None,
        None,
        [u64::from(total), wide_rt::FILL_ZERO, 0, 0],
    );
    let mut offset = total;
    for v in parts {
        offset -= v.width;
        call(
            cx,
            WideOp::Insert,
            Some(dst),
            Some(v.ptr),
            None,
            [u64::from(v.width), u64::from(total), u64::from(offset), 0],
        );
        release(v.ptr);
    }
    let image = HugeValue {
        ptr: dst,
        width: total,
        signed: false,
    };
    match e.streaming_slice_size() {
        Some(n) if n > 0 => Ok(reorder_slices(cx, image, n, false)),
        _ => Ok(image),
    }
}

/// `{<<n{}}` on the stream image `v`: `n`-bit slices from the least
/// significant end, the first one most significant (`inverse`: from the most
/// significant end, the first one least significant). Consumes `v`.
fn reorder_slices<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    v: HugeValue<'ctx>,
    n: u64,
    inverse: bool,
) -> HugeValue<'ctx> {
    let dst = temp(cx, v.width);
    call(
        cx,
        WideOp::ReverseSlices,
        Some(dst),
        Some(v.ptr),
        None,
        [u64::from(v.width), v.info(), n, u64::from(inverse)],
    );
    release(v.ptr);
    HugeValue {
        ptr: dst,
        width: v.width,
        signed: v.signed,
    }
}

/// `{<<n{a, b}} = value` into plain operands when the source or an operand is
/// huge: the source's leading bits, un-reordered, split across the operands
/// most significant first (LRM 11.4.14.2). `None` (nothing emitted) for any
/// other stream target.
fn store_stream<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    lhs: sv_lang::Expression<'d>,
    value: LoweredValue<'ctx>,
) -> Result<Option<()>, LoweringError> {
    let streams = lhs.streams();
    let operands: Vec<_> = streams.iter().map(|s| s.operand()).collect();
    let plain = !operands.is_empty()
        && streams.iter().all(|s| s.with_expr().is_none())
        && operands.iter().all(|o| {
            o.kind() != ExpressionKind::Streaming
                && o.expr_type().is_some_and(|t| plain_stream_type(&t))
        });
    let any_huge =
        matches!(value, LoweredValue::Huge(_)) || operands.iter().any(|o| is_huge_expr(*o));
    if !plain || !any_huge {
        return Ok(None);
    }
    require_four_state(cx)?;
    let src = to_huge(
        cx,
        value,
        "the value unpacked into a stream target must be integral",
    )?;
    let widths = operands
        .iter()
        .map(|o| type_width(cx, *o))
        .collect::<Result<Vec<_>, _>>()?;
    let total: u32 = widths.iter().sum();
    if src.width < total {
        return loud(
            cx,
            &format!(
                "a {}-bit source unpacked into a {total}-bit stream",
                src.width
            ),
        );
    }
    let lead = HugeValue {
        ptr: extract_new(
            cx,
            src.ptr,
            src.width,
            total,
            c64(cx, u64::from(src.width - total)),
            None,
        ),
        width: total,
        signed: false,
    };
    release(src.ptr);
    let lead = match lhs.streaming_slice_size() {
        Some(n) if n > 0 => reorder_slices(cx, lead, n, true),
        _ => lead,
    };
    let mut top = total;
    for (operand, w) in operands.into_iter().zip(widths) {
        top -= w;
        let piece = extract_new(cx, lead.ptr, total, w, c64(cx, u64::from(top)), None);
        let lv = finish(cx, piece, w, type_signed(operand));
        crate::expr::store_to(cx, operand, lv)?;
        discard(lv);
    }
    release(lead.ptr);
    Ok(Some(()))
}

fn conversion<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let operand = e
        .conversion_operand()
        .ok_or_else(|| LoweringError::unsupported_at("conversion without operand", &cx.fn_sym))?;
    let (Some(to_ty), Some(from_ty)) = (e.expr_type(), operand.expr_type()) else {
        return loud(cx, "conversion without types");
    };
    let to_w = to_ty.value_width();
    let to_signed = to_ty.is_signed();
    // A string is its bytes (LRM 6.16), whatever the conversion's kind.
    if to_ty.is_string() {
        let s = crate::strings::to_string(cx, lower_expr_h(cx, operand)?)?;
        return Ok(LoweredValue::Str(s));
    }
    if from_ty.is_string() {
        let s = crate::strings::to_string(cx, lower_expr_h(cx, operand)?)?;
        return Ok(from_string(cx, s, to_w, to_signed));
    }
    let kind = e
        .conversion_kind()
        .ok_or_else(|| LoweringError::unsupported_at("conversion without kind", &cx.fn_sym))?;
    match crate::expr::classify_conversion_kind(kind)
        .map_err(|msg| LoweringError::unsupported_at(&msg, &cx.fn_sym))?
    {
        crate::expr::ConversionAction::Value => {}
        crate::expr::ConversionAction::Stream => {
            // The stream, left-aligned in the target (zero-filled on the
            // right).
            let s = stream_image(cx, operand)?;
            if s.width > to_w {
                return loud(
                    cx,
                    &format!("a {}-bit stream into a {to_w}-bit target", s.width),
                );
            }
            if s.width == to_w {
                return Ok(LoweredValue::Huge(HugeValue {
                    ptr: s.ptr,
                    width: to_w,
                    signed: to_signed,
                }));
            }
            let dst = temp(cx, to_w);
            call(
                cx,
                WideOp::Fill,
                Some(dst),
                None,
                None,
                [u64::from(to_w), wide_rt::FILL_ZERO, 0, 0],
            );
            call(
                cx,
                WideOp::Insert,
                Some(dst),
                Some(s.ptr),
                None,
                [
                    u64::from(s.width),
                    u64::from(to_w),
                    u64::from(to_w - s.width),
                    0,
                ],
            );
            release(s.ptr);
            return Ok(finish(cx, dst, to_w, to_signed));
        }
        crate::expr::ConversionAction::BitstreamCast => {
            // The operand's bits, read back as the target type.
            let v = stream_image(cx, operand)?;
            if v.width != to_w {
                return loud(
                    cx,
                    &format!("a bit-stream cast of {} bits to a {to_w}-bit type", v.width),
                );
            }
            return Ok(LoweredValue::Huge(HugeValue {
                ptr: v.ptr,
                width: to_w,
                signed: to_signed,
            }));
        }
    }
    // A real converts by value (LRM 6.24.1).
    if from_ty.is_floating() {
        if !to_ty.is_integral() {
            return loud(cx, "a conversion from a real to a non-integral type");
        }
        let rv = lower_expr_h(cx, operand)?.into_real(
            "real-to-int conversion operand must itself lower to a real value",
            &cx.fn_sym,
        )?;
        let f = crate::real::as_f64(cx, rv.llvm);
        let bits =
            cx.b.build_bit_cast(f, cx.ctx.i64_type(), "huge_real_bits")
                .unwrap()
                .into_int_value();
        let dst = temp(cx, to_w);
        emit(
            cx,
            WideOp::FromReal,
            Some(dst),
            None,
            None,
            [c64(cx, u64::from(to_w)), bits, c64(cx, 0), c64(cx, 0)],
        );
        return Ok(finish(cx, dst, to_w, to_signed));
    }
    if to_ty.is_floating() {
        if !from_ty.is_integral() {
            return loud(cx, "a conversion from a non-integral type to a real");
        }
        let v = lower_operand(cx, operand)?;
        let mant = if to_ty.bit_width() == 32 { 24 } else { 53 };
        let bits = call(
            cx,
            WideOp::ToReal,
            None,
            Some(v.ptr),
            None,
            [0, v.info(), mant, 0],
        );
        release(v.ptr);
        let f =
            cx.b.build_bit_cast(bits, cx.ctx.f64_type(), "huge_real")
                .unwrap()
                .into_float_value();
        return Ok(LoweredValue::Real(crate::value::RealValue { llvm: f }));
    }
    let v = lower_operand(cx, operand)?;
    let flatten = from_ty.is_four_state() && !to_ty.is_four_state();
    if !flatten && to_w == v.width {
        return Ok(LoweredValue::Huge(HugeValue {
            ptr: v.ptr,
            width: to_w,
            signed: to_signed,
        }));
    }
    // The extension follows the operand's own signedness; `Flatten` also
    // resizes (it reads its operand at `to_w`).
    let from_info = wide_rt::info(v.width, from_ty.is_signed());
    release(v.ptr);
    let dst = temp(cx, to_w);
    call(
        cx,
        if flatten {
            WideOp::Flatten
        } else {
            WideOp::Resize
        },
        Some(dst),
        Some(v.ptr),
        None,
        [u64::from(to_w), from_info, 0, 0],
    );
    Ok(finish(cx, dst, to_w, to_signed))
}

/// The string `s` as a `width`-bit value (LRM 6.16): its last character in
/// the low byte, bytes past `width` dropped.
pub(crate) fn from_string<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    s: crate::strings::StrValue<'ctx>,
    width: u32,
    signed: bool,
) -> LoweredValue<'ctx> {
    let dst = temp(cx, width);
    emit(
        cx,
        WideOp::FromStr,
        Some(dst),
        None,
        None,
        [c64(cx, u64::from(width)), s.llvm, c64(cx, 0), c64(cx, 0)],
    );
    finish(cx, dst, width, signed)
}

/// The huge value `h` as a string (LRM 6.16): its bytes, most significant
/// first, NUL bytes dropped. Consumes `h`.
pub(crate) fn to_string<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    h: HugeValue<'ctx>,
) -> crate::strings::StrValue<'ctx> {
    let id = call(
        cx,
        WideOp::ToStr,
        None,
        Some(h.ptr),
        None,
        [0, h.info(), 0, 0],
    );
    release(h.ptr);
    crate::strings::StrValue { llvm: id }
}

/// The current value of a compound assignment's target (`a += b`): a fresh
/// copy, so the operator that consumes it may release it.
fn lvalue_reference<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let Some(LoweredValue::Huge(h)) = cx.lvalue_ref.get() else {
        return loud(
            cx,
            "internal: a compound-assignment reference of a huge type outside a huge assignment",
        );
    };
    let dst = temp(cx, h.width);
    call(
        cx,
        WideOp::Resize,
        Some(dst),
        Some(h.ptr),
        None,
        [u64::from(h.width), h.info(), 0, 0],
    );
    Ok(LoweredValue::Huge(HugeValue {
        ptr: dst,
        width: h.width,
        signed: h.signed,
    }))
}

// ---------------------------------------------------------------------------
// Select chains

/// One level of a select chain: the window it addresses inside its parent.
#[derive(Clone, Copy)]
struct Level<'ctx> {
    /// Bit offset of the window's least significant bit in the parent (`i64`;
    /// negative for a window straddling the parent's low edge).
    start: IntValue<'ctx>,
    /// An `i1`: the window is invalid (unknown index, or wholly outside the
    /// parent): a read yields X, a write is ignored.
    bad: Option<IntValue<'ctx>>,
    width: u32,
    parent_width: u32,
    /// The window lies entirely inside its parent whenever it is not `bad`.
    exact: bool,
}

/// The span of a huge signal's value a write may have changed: `width` bits
/// from bit `start` (an `i64`, negative when the window straddles the low
/// edge), none at all when `bad`. The runtime clips it to the signal.
#[derive(Clone, Copy)]
struct WriteWindow<'ctx> {
    start: IntValue<'ctx>,
    width: u64,
    bad: Option<IntValue<'ctx>>,
}

fn const_i64<'ctx>(cx: &LowerCtx<'ctx, '_, '_>, v: i64) -> IntValue<'ctx> {
    cx.ctx.i64_type().const_int(v as u64, true)
}

/// A select index as a [`PackedValue`]: an ordinary narrow index as is; a
/// huge one (wider than the threshold) clamped by the runtime to a signed
/// 64-bit value (any index past +-2^62 is out of range of every window anyway)
/// that is unknown exactly when the index has an X/Z bit.
fn index_value<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    what: &str,
) -> Result<PackedValue<'ctx>, LoweringError> {
    let lv = lower_expr_h(cx, e)?;
    let LoweredValue::Huge(h) = lv else {
        return lv.into_packed(what, &cx.fn_sym);
    };
    let r = call(
        cx,
        WideOp::Index,
        None,
        Some(h.ptr),
        None,
        [0, h.info(), 0, 0],
    );
    release(h.ptr);
    let i64t = cx.ctx.i64_type();
    let unknown =
        cx.b.build_int_compare(
            IntPredicate::EQ,
            r,
            i64t.const_int(i64::MIN as u64, true),
            "hidx_x",
        )
        .unwrap();
    let val =
        cx.b.build_select(unknown, i64t.const_zero(), r, "hidx_val")
            .unwrap()
            .into_int_value();
    let unk =
        cx.b.build_select(
            unknown,
            i64t.const_all_ones(),
            i64t.const_zero(),
            "hidx_unk",
        )
        .unwrap()
        .into_int_value();
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    Ok(op.pack(val, unk, 64, ValueRepr::FourState, true))
}

/// A level for a part-select with constant bounds, or an `[i +: n]` /
/// `[i -: n]` / `[i]` with a dynamic index. Mirrors `select::select_window`.
fn select_level<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    geom: SelectGeom,
    parent_width: u32,
) -> Result<Level<'ctx>, LoweringError> {
    use crate::expr::{RangeSelectionKindClass, classify_range_selection_kind};
    match e.kind() {
        ExpressionKind::ElementSelect => {
            let idx_expr = e.selector().ok_or_else(|| {
                LoweringError::unsupported_at("bit-select missing index", &cx.fn_sym)
            })?;
            let idx = index_value(cx, idx_expr, "a bit-select index must be an integral value")?;
            dynamic_level(cx, idx, 0, 1, geom, parent_width)
        }
        ExpressionKind::RangeSelect => {
            let kind_ord = e.range_selection_kind().ok_or_else(|| {
                LoweringError::unsupported_at(
                    "part-select missing range_selection_kind",
                    &cx.fn_sym,
                )
            })?;
            let kind = classify_range_selection_kind(kind_ord)
                .map_err(|msg| LoweringError::unsupported_at(&msg, &cx.fn_sym))?;
            let (Some(left), Some(right)) = (e.range_left(), e.range_right()) else {
                return loud(cx, "part-select missing a bound");
            };
            match kind {
                RangeSelectionKindClass::Simple => {
                    let l = require_const_i64(
                        left,
                        "simple part-select left bound must be a compile-time constant",
                        &cx.fn_sym,
                    )?;
                    let r = require_const_i64(
                        right,
                        "simple part-select right bound must be a compile-time constant",
                        &cx.fn_sym,
                    )?;
                    let (pl, pr) = (geom.pos(l), geom.pos(r));
                    let (lo_e, hi_e) = (pl.min(pr), pl.max(pr));
                    let count = hi_e - lo_e + 1;
                    let width = checked_window_width(cx, count, geom, parent_width)?;
                    let in_range = lo_e >= 0 && hi_e < geom.elems();
                    let fully_out = hi_e < 0 || lo_e >= geom.elems();
                    let ew = i64::from(geom.elem_width);
                    let start = if fully_out { 0 } else { lo_e * ew };
                    Ok(Level {
                        start: const_i64(cx, start),
                        bad: fully_out.then(|| cx.ctx.bool_type().const_int(1, false)),
                        width,
                        parent_width,
                        exact: in_range,
                    })
                }
                RangeSelectionKindClass::IndexedUp | RangeSelectionKindClass::IndexedDown => {
                    let count = require_const_i64(
                        right,
                        "indexed part-select width must be a compile-time constant",
                        &cx.fn_sym,
                    )?;
                    if count <= 0 {
                        return loud(cx, "indexed part-select with a non-positive width");
                    }
                    checked_window_width(cx, count, geom, parent_width)?;
                    let idx = index_value(
                        cx,
                        left,
                        "an indexed part-select base must be an integral value",
                    )?;
                    let up = kind == RangeSelectionKindClass::IndexedUp;
                    let low_delta = match (up, geom.descending()) {
                        (true, true) | (false, false) => 0,
                        (true, false) => count - 1,
                        (false, true) => -(count - 1),
                    };
                    dynamic_level(cx, idx, low_delta, count, geom, parent_width)
                }
            }
        }
        other => Err(LoweringError::unsupported_at(
            &format!("internal: select_level on a {other:?} expression"),
            &cx.fn_sym,
        )),
    }
}

/// The window of `count` elements whose least significant element has declared
/// index `idx + low_delta`, as a [`Level`] (`select::dynamic_window` with a
/// signed 64-bit start instead of a clamped base-width one).
fn dynamic_level<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    idx: PackedValue<'ctx>,
    low_delta: i64,
    count: i64,
    geom: SelectGeom,
    parent_width: u32,
) -> Result<Level<'ctx>, LoweringError> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    // Wide enough that the index (at its own signedness), the declared bounds
    // (i32) and the element scaling can never overflow.
    let cw = idx.width.max(64) + 2;
    let cty = op.i_n(cw);
    let val = op.lane_val(idx);
    let ext = if idx.width == cw {
        val
    } else if idx.signed {
        cx.b.build_int_s_extend(val, cty, "hsel_idx_sext").unwrap()
    } else {
        cx.b.build_int_z_extend(val, cty, "hsel_idx_zext").unwrap()
    };
    let c = |v: i64| cty.const_int(v as u64, true);
    let low =
        cx.b.build_int_add(ext, c(low_delta), "hsel_low_index")
            .unwrap();
    let pos = if geom.descending() {
        cx.b.build_int_sub(low, c(geom.right), "hsel_pos").unwrap()
    } else {
        cx.b.build_int_sub(c(geom.right), low, "hsel_pos").unwrap()
    };
    let cmp =
        |p: IntPredicate, v: i64, name: &str| cx.b.build_int_compare(p, pos, c(v), name).unwrap();
    // A single element is either in or out; a wider window may straddle an
    // edge and then reads/writes just the bits that are in range.
    let fully_out = if count > 1 {
        let before = cmp(IntPredicate::SLT, 1 - count, "hsel_before");
        let past = cmp(IntPredicate::SGE, geom.elems(), "hsel_past");
        cx.b.build_or(before, past, "hsel_fully_out").unwrap()
    } else {
        let below = cmp(IntPredicate::SLT, 0, "hsel_below");
        let above = cmp(IntPredicate::SGT, geom.elems() - 1, "hsel_above");
        cx.b.build_or(below, above, "hsel_oob").unwrap()
    };
    let unk = op.lane_unk(idx);
    let unknown =
        cx.b.build_int_compare(
            IntPredicate::NE,
            unk,
            unk.get_type().const_zero(),
            "hsel_idx_unknown",
        )
        .unwrap();
    let bad = cx.b.build_or(fully_out, unknown, "hsel_bad").unwrap();
    let ew = c(i64::from(geom.elem_width));
    let start = cx.b.build_int_mul(pos, ew, "hsel_start_bits").unwrap();
    let start =
        cx.b.build_select(bad, c(0), start, "hsel_start_safe")
            .unwrap()
            .into_int_value();
    let start =
        cx.b.build_int_cast_sign_flag(start, cx.ctx.i64_type(), true, "hsel_start64")
            .unwrap();
    Ok(Level {
        start,
        bad: Some(bad),
        width: (count as u32) * geom.elem_width,
        parent_width,
        exact: count == 1,
    })
}

/// A struct/union field as a constant window.
fn member_level<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    base: sv_lang::Expression<'d>,
    parent_width: u32,
) -> Result<Level<'ctx>, LoweringError> {
    let member = e.member_symbol().ok_or_else(|| {
        LoweringError::unsupported_at("member access without a member", &cx.fn_sym)
    })?;
    let Some((off, fw)) = field_window(base.expr_type(), &member)
        .filter(|(off, fw)| *fw > 0 && fw + off <= u64::from(parent_width))
    else {
        return loud(
            cx,
            "a struct/union field of a vector wider than 65535 bits that is not a slice of its \
             base's stored bits",
        );
    };
    Ok(Level {
        start: const_i64(cx, off as i64),
        bad: None,
        width: fw as u32,
        parent_width,
        exact: true,
    })
}

/// Walks a select chain down to its root, innermost level last. The indices
/// are lowered (evaluated) base first.
fn resolve<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<(sv_lang::Expression<'d>, Vec<Level<'ctx>>), LoweringError> {
    let Some(base) = chain_base(e) else {
        return Ok((e, Vec::new()));
    };
    let (root, mut levels) = resolve(cx, base)?;
    let parent_width = type_width(cx, base)?;
    // `chain_base` returned `Some`, so `e` is a member access or a select;
    // `select_level` rejects anything else loudly.
    let level = if e.kind() == ExpressionKind::MemberAccess {
        member_level(cx, e, base, parent_width)?
    } else {
        let geom = geom_of(base.expr_type(), parent_width);
        select_level(cx, e, geom, parent_width)?
    };
    levels.push(level);
    Ok((root, levels))
}

fn or_bad<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    a: Option<IntValue<'ctx>>,
    b: Option<IntValue<'ctx>>,
) -> Option<IntValue<'ctx>> {
    match (a, b) {
        (Some(a), Some(b)) => Some(cx.b.build_or(a, b, "hbad").unwrap()),
        (x, None) => x,
        (None, y) => y,
    }
}

fn bad_i64<'ctx>(cx: &LowerCtx<'ctx, '_, '_>, bad: Option<IntValue<'ctx>>) -> IntValue<'ctx> {
    match bad {
        Some(b) => {
            cx.b.build_int_z_extend(b, cx.ctx.i64_type(), "hbad64")
                .unwrap()
        }
        None => cx.ctx.i64_type().const_zero(),
    }
}

/// The summed start offset and combined invalid flag of `levels`.
fn fold<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    levels: &[Level<'ctx>],
) -> (IntValue<'ctx>, Option<IntValue<'ctx>>) {
    let mut start = const_i64(cx, 0);
    let mut bad = None;
    for l in levels {
        start = cx.b.build_int_add(start, l.start, "hstart").unwrap();
        bad = or_bad(cx, bad, l.bad);
    }
    (start, bad)
}

/// `Extract` of a `width`-bit window of the `src_width`-bit image `src` into
/// a new temporary.
fn extract_new<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    src: PointerValue<'ctx>,
    src_width: u32,
    width: u32,
    start: IntValue<'ctx>,
    bad: Option<IntValue<'ctx>>,
) -> PointerValue<'ctx> {
    let dst = temp(cx, width);
    emit(
        cx,
        WideOp::Extract,
        Some(dst),
        Some(src),
        None,
        [
            c64(cx, u64::from(width)),
            c64(cx, wide_rt::info(src_width, false)),
            start,
            bad_i64(cx, bad),
        ],
    );
    dst
}

/// `Insert` of the `src_width`-bit image `src` into the `dst_width`-bit image
/// `dst` at `start` (clipped to `dst`; ignored when `bad`).
fn insert_into<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    dst: PointerValue<'ctx>,
    dst_width: u32,
    src: PointerValue<'ctx>,
    src_width: u32,
    start: IntValue<'ctx>,
    bad: Option<IntValue<'ctx>>,
) {
    emit(
        cx,
        WideOp::Insert,
        Some(dst),
        Some(src),
        None,
        [
            c64(cx, u64::from(src_width)),
            c64(cx, u64::from(dst_width)),
            start,
            bad_i64(cx, bad),
        ],
    );
}

fn read_chain<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    let (root, levels) = resolve(cx, e)?;
    let Some(last) = levels.last() else {
        if is_huge_expr(e) && crate::classes::is_class_field_lvalue(e) {
            return read_class_prop(cx, e);
        }
        if mem_elem_root(cx, e) {
            return read_mem_elem(cx, e);
        }
        // `e` is a select of something that is not a vector (an element of an
        // unpacked array of huge elements): the ordinary read.
        return fall_back(cx, e);
    };
    let width = last.width;
    if type_width(cx, e)? != width {
        return loud(
            cx,
            "internal: a select's type width differs from its resolved window",
        );
    }
    let mut cur = lower_operand(cx, root)?;
    let mut start = const_i64(cx, 0);
    let mut bad = None;
    for (i, level) in levels.iter().enumerate() {
        start = cx.b.build_int_add(start, level.start, "hstart").unwrap();
        bad = or_bad(cx, bad, level.bad);
        // A level followed by one that may reach outside it (or the last
        // level) is read out; an exact level just offsets the next one.
        if i + 1 == levels.len() || !levels[i + 1].exact {
            release(cur.ptr);
            let dst = extract_new(cx, cur.ptr, cur.width, level.width, start, bad);
            cur = HugeValue {
                ptr: dst,
                width: level.width,
                signed: false,
            };
            start = const_i64(cx, 0);
            bad = None;
        }
    }
    let out = finish(cx, cur.ptr, width, type_signed(e));
    // A select of a 2-state operand reads 0, not x, out of range (LRM 11.5.1).
    Ok(match out {
        LoweredValue::Packed(v) => {
            let op = OpBuilder {
                ctx: cx.ctx,
                b: &cx.b,
            };
            LoweredValue::Packed(crate::select::two_state_result(&op, e, v))
        }
        other => other,
    })
}

// ---------------------------------------------------------------------------
// Writes

/// The shadow `alloca` of the huge local named by `root`.
fn root_shadow<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    root: sv_lang::Expression<'d>,
) -> Result<(PointerValue<'ctx>, u32), LoweringError> {
    if !vj_sim::symid::is_value_name(root.kind()) {
        return loud(
            cx,
            "an assignment to a vector wider than 65535 bits through something other than a \
             named variable",
        );
    }
    let sym = root
        .referenced_symbol()
        .ok_or_else(|| LoweringError::unsupported_at("lvalue without a symbol", &cx.fn_sym))?;
    match cx.locals.get(&sym.uid()) {
        Some(&(ptr, width, _)) if is_huge_width(width) => Ok((ptr, width)),
        _ => Err(LoweringError::unsupported_at(
            "an assignment to a vector wider than 65535 bits that is not a module signal of \
             this process (a procedural, function or class-property variable of that width)",
            &sym,
        )),
    }
}

/// Writes `v` through `levels` into the `dst_width`-bit image `dst`: a
/// read-modify-write chain. A level that may reach outside its parent (a
/// dynamic multi-element window, an out-of-range constant part-select) is
/// extracted as a value, patched by the levels below it and inserted back, so
/// the bits that fall outside are dropped exactly once.
///
/// Returns the span of `dst` that may have changed (a [`WriteWindow`], an
/// over-approximation where a straddling window is read-modified-written).
fn write_levels<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    dst: PointerValue<'ctx>,
    dst_width: u32,
    levels: &[Level<'ctx>],
    v: HugeValue<'ctx>,
) -> Result<WriteWindow<'ctx>, LoweringError> {
    let Some(k) = levels.iter().position(|l| !l.exact) else {
        let (start, bad) = fold(cx, levels);
        insert_into(cx, dst, dst_width, v.ptr, v.width, start, bad);
        return Ok(WriteWindow {
            start,
            width: u64::from(v.width),
            bad,
        });
    };
    let l = levels[k];
    let (prefix_start, prefix_bad) = fold(cx, &levels[..k]);
    // Level `k`'s own window when it patches `dst` directly, else the parent
    // window level `k` lives in, which is inserted back whole.
    let window = if k == 0 {
        WriteWindow {
            start: l.start,
            width: u64::from(l.width),
            bad: l.bad,
        }
    } else {
        WriteWindow {
            start: prefix_start,
            width: u64::from(l.parent_width),
            bad: prefix_bad,
        }
    };
    // The parent window of level `k`, inside `dst`.
    let parent = if k == 0 {
        dst
    } else {
        extract_new(cx, dst, dst_width, l.parent_width, prefix_start, prefix_bad)
    };
    if k + 1 == levels.len() {
        insert_into(cx, parent, l.parent_width, v.ptr, v.width, l.start, l.bad);
    } else {
        let q = extract_new(cx, parent, l.parent_width, l.width, l.start, l.bad);
        // The inner window is relative to `q`; the whole of `q` is inserted
        // back, so `window` already covers it.
        write_levels(cx, q, l.width, &levels[k + 1..], v)?;
        insert_into(cx, parent, l.parent_width, q, l.width, l.start, l.bad);
        release(q);
    }
    if k > 0 {
        insert_into(
            cx,
            dst,
            dst_width,
            parent,
            l.parent_width,
            prefix_start,
            prefix_bad,
        );
        release(parent);
    }
    Ok(window)
}

/// Stores `v` into the lvalue `lhs` (a huge module signal's shadow, whole or
/// through a select chain). Returns the root's shadow pointer and width.
fn write_chain<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    lhs: sv_lang::Expression<'d>,
    v: HugeValue<'ctx>,
) -> Result<(PointerValue<'ctx>, u32), LoweringError> {
    let (root, levels) = resolve(cx, lhs)?;
    let (shadow, width) = root_shadow(cx, root)?;
    match levels.last() {
        None => {
            call(
                cx,
                WideOp::Resize,
                Some(shadow),
                Some(v.ptr),
                None,
                [u64::from(width), v.info(), 0, 0],
            );
            mark_window(cx, shadow, None);
        }
        Some(last) => {
            if last.width != v.width {
                return loud(
                    cx,
                    "internal: the value assigned to a select of a huge vector is not as wide \
                     as the select",
                );
            }
            let window = write_levels(cx, shadow, width, &levels, v)?;
            mark_window(cx, shadow, Some(window));
        }
    }
    Ok((shadow, width))
}

/// Records, in the runtime's dirty window of the huge module signal whose
/// shadow is `shadow`, the span a write just touched (`None`: all of it). A
/// shadow that is not written through (a function-local copy) has no window.
fn mark_window<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    shadow: PointerValue<'ctx>,
    window: Option<WriteWindow<'ctx>>,
) {
    let Some(wt) = cx.write_through.as_ref() else {
        return;
    };
    let Some(&(slot, width, _)) = wt.signals.get(&shadow) else {
        return;
    };
    let i64t = cx.ctx.i64_type();
    let (start, len) = match window {
        None => (i64t.const_zero(), i64t.const_int(u64::from(width), false)),
        Some(w) => {
            let len = i64t.const_int(w.width, false);
            let len = match w.bad {
                Some(bad) => {
                    cx.b.build_select(bad, i64t.const_zero(), len, "hmark_len")
                        .unwrap()
                        .into_int_value()
                }
                None => len,
            };
            (w.start, len)
        }
    };
    rt_mark(cx, wt.handle, slot, width, start, len);
}

fn assignment<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<LoweredValue<'ctx>, LoweringError> {
    if e.is_nonblocking() == Some(true) {
        // A class property inside a subroutine body (`p <= v` in a method): staged
        // against the object like the same statement in a process.
        if let (Some(lhs), Some(rhs)) = (e.left(), e.right())
            && class_prop_root(lhs)
            && !huge_local_root(cx, lhs)
        {
            let delay = crate::process_lower::nba_intra_delay(cx, e)?
                .unwrap_or_else(|| cx.ctx.i64_type().const_zero());
            let rhs = crate::collections::assigned_value(cx, lhs, rhs)?;
            lower_nba_class_prop(cx, lhs, rhs, delay)?;
            let z = cx.ctx.bool_type().const_zero();
            let op = OpBuilder {
                ctx: cx.ctx,
                b: &cx.b,
            };
            return Ok(LoweredValue::Packed(op.pack(z, z, 1, cx.repr, false)));
        }
        return loud(
            cx,
            "a nonblocking assignment to a vector wider than 65535 bits outside a process \
             statement",
        );
    }
    let (Some(lhs), Some(rhs)) = (e.left(), e.right()) else {
        return loud(cx, "assignment without both sides");
    };
    let rhs_value = if e.is_compound_assignment() == Some(true) {
        let current = lower_expr_h(cx, lhs)?;
        let saved = cx.lvalue_ref.replace(Some(current));
        let r = lower_expr_h(cx, rhs);
        cx.lvalue_ref.set(saved);
        if let LoweredValue::Huge(h) = current {
            release(h.ptr);
        }
        r?
    } else {
        lower_expr_h(cx, rhs)?
    };
    let v = to_huge(
        cx,
        rhs_value,
        "the value assigned to a vector wider than 65535 bits must be integral",
    )?;
    if class_prop_root(lhs) && !huge_local_root(cx, lhs) {
        write_class_prop(cx, lhs, v)?;
        return Ok(finish(cx, v.ptr, v.width, v.signed));
    }
    if mem_elem_root(cx, lhs) {
        write_mem_elem(cx, lhs, v)?;
        return Ok(finish(cx, v.ptr, v.width, v.signed));
    }
    let (shadow, width) = write_chain(cx, lhs, v)?;
    write_through_commit(cx, shadow);
    if chain_base(lhs).is_none() {
        release(v.ptr);
        let signed = cx
            .locals
            .values()
            .find(|(p, _, _)| *p == shadow)
            .is_some_and(|(_, _, s)| *s);
        return Ok(LoweredValue::Huge(HugeValue {
            ptr: shadow,
            width,
            signed,
        }));
    }
    Ok(finish(cx, v.ptr, v.width, v.signed))
}

/// The `store_to` hook: `Some(())` when `lhs` is rooted in a huge vector and
/// `value` has been stored to it.
pub(crate) fn try_store<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    lhs: sv_lang::Expression<'d>,
    value: LoweredValue<'ctx>,
) -> Result<Option<()>, LoweringError> {
    if lhs.kind() == ExpressionKind::Streaming {
        return store_stream(cx, lhs, value);
    }
    if class_prop_root(lhs) && !huge_local_root(cx, lhs) {
        require_four_state(cx)?;
        let v = to_huge(
            cx,
            value,
            "the value stored to a vector wider than the huge threshold must be integral",
        )?;
        write_class_prop(cx, lhs, v)?;
        release(v.ptr);
        return Ok(Some(()));
    }
    if mem_elem_root(cx, lhs) {
        require_four_state(cx)?;
        let v = to_huge(
            cx,
            value,
            "the value stored to a vector wider than the huge threshold must be integral",
        )?;
        write_mem_elem(cx, lhs, v)?;
        release(v.ptr);
        return Ok(Some(()));
    }
    if !rooted_in_huge(lhs) || !huge_local_root(cx, lhs) {
        return Ok(None);
    }
    require_four_state(cx)?;
    let v = to_huge(
        cx,
        value,
        "the value stored to a vector wider than the huge threshold must be integral",
    )?;
    let (shadow, _) = write_chain(cx, lhs, v)?;
    write_through_commit(cx, shadow);
    release(v.ptr);
    Ok(Some(()))
}

/// Stores `value` to the whole variable `sym` when that is a huge module
/// signal's shadow; `false` (nothing emitted) when it is not.
pub(crate) fn try_store_symbol<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    sym: &sv_lang::Symbol<'_>,
    value: LoweredValue<'ctx>,
) -> Result<bool, LoweringError> {
    let Some(&(shadow, width, _)) = cx.locals.get(&sym.uid()) else {
        return Ok(false);
    };
    if !is_huge_width(width) {
        return Ok(false);
    }
    require_four_state(cx)?;
    let v = to_huge(
        cx,
        value,
        "the value stored to a vector wider than 65535 bits must be integral",
    )?;
    call(
        cx,
        WideOp::Resize,
        Some(shadow),
        Some(v.ptr),
        None,
        [u64::from(width), v.info(), 0, 0],
    );
    write_through_store(cx, shadow);
    release(v.ptr);
    Ok(true)
}

/// Releases the temporary of a value nothing will read (an expression
/// statement's result).
pub(crate) fn discard(lv: LoweredValue<'_>) {
    if let LoweredValue::Huge(h) = lv {
        release(h.ptr);
    }
}

/// Stores `value` (as wide as `width`) to the module signal in `slot` of
/// `handle` through the accessor ABI.
pub(crate) fn push_planes<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    h: HugeValue<'ctx>,
    aval: PointerValue<'ctx>,
    bval: PointerValue<'ctx>,
) {
    call_wide(
        cx,
        WideOp::ToPlanes,
        aval,
        h.ptr,
        bval,
        [u64::from(h.width), 0, 0, 0],
    );
}

// ---------------------------------------------------------------------------
// Nonblocking assignment

/// Stages `lhs <= rhs` for an lvalue rooted in a huge module signal: the
/// written bits as a value image plus a mask image of the bits written, handed
/// to `sv_rt_nba_masked` (applied in the NBA region).
pub(crate) fn lower_nba<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    handle: PointerValue<'ctx>,
    signals: &std::collections::HashMap<vj_sim::symid::SymbolId, crate::process_lower::SignalSlot>,
    lhs: sv_lang::Expression<'d>,
    rhs: LoweredValue<'ctx>,
    delay: IntValue<'ctx>,
) -> Result<(), LoweringError> {
    require_four_state(cx)?;
    let (root, levels) = resolve(cx, lhs)?;
    if !vj_sim::symid::is_value_name(root.kind()) {
        return loud(
            cx,
            "a nonblocking assignment to a vector wider than 65535 bits through something \
             other than a named signal",
        );
    }
    let sym = root
        .referenced_symbol()
        .ok_or_else(|| LoweringError::unsupported_at("lvalue without a symbol", &cx.fn_sym))?;
    let info = signals.get(&sym.uid()).copied().ok_or_else(|| {
        LoweringError::unsupported_at(
            "a nonblocking assignment to a process-local variable (only a module-level signal \
             has an NBA staging target)",
            &sym,
        )
    })?;
    let slot = cx
        .specify_dest
        .get(&sym.uid())
        .map_or(info.slot, |r| r.slot);
    let w = info.width;
    if !is_huge_width(w) {
        return loud(cx, "internal: a narrow signal reached the huge NBA staging");
    }
    let v = to_huge(
        cx,
        rhs,
        "the value assigned to a vector wider than 65535 bits must be integral",
    )?;
    // A write to a select whose window is known to lie inside the signal (every
    // level exact), or to a select of the signal itself (a lone level, which
    // the runtime clips to the signal): stage just the words it covers.
    if let Some(last) = levels.last()
        && (levels.len() == 1 || levels.iter().all(|l| l.exact))
    {
        if last.width != v.width {
            return loud(
                cx,
                "internal: the value assigned to a select of a huge vector is not as wide \
                 as the select",
            );
        }
        let (start, bad) = fold(cx, &levels);
        let func = cx.b.get_insert_block().unwrap().get_parent().unwrap();
        let done_bb = cx.ctx.append_basic_block(func, "hnba_done");
        if let Some(bad) = bad {
            let stage_bb = cx.ctx.append_basic_block(func, "hnba_stage");
            cx.b.build_conditional_branch(bad, done_bb, stage_bb)
                .unwrap();
            cx.b.position_at_end(stage_bb);
        }
        rt_nba_window(cx, handle, slot, w, start, last.width, v.ptr, delay);
        release(v.ptr);
        cx.b.build_unconditional_branch(done_bb).unwrap();
        cx.b.position_at_end(done_bb);
        return Ok(());
    }
    let image = temp(cx, w);
    let mask = temp(cx, w);
    let zero = |p: PointerValue<'ctx>| {
        call(
            cx,
            WideOp::Fill,
            Some(p),
            None,
            None,
            [u64::from(w), wide_rt::FILL_ZERO, 0, 0],
        );
    };
    zero(image);
    zero(mask);
    let mut bad = None;
    match levels.last() {
        None => {
            call(
                cx,
                WideOp::Resize,
                Some(image),
                Some(v.ptr),
                None,
                [u64::from(w), v.info(), 0, 0],
            );
            call(
                cx,
                WideOp::Fill,
                Some(mask),
                None,
                None,
                [u64::from(w), wide_rt::FILL_ONE, 0, 0],
            );
        }
        Some(last) => {
            if last.width != v.width {
                return loud(
                    cx,
                    "internal: the value assigned to a select of a huge vector is not as wide \
                     as the select",
                );
            }
            for l in &levels {
                bad = or_bad(cx, bad, l.bad);
            }
            write_levels(cx, image, w, &levels, v)?;
            let ones = temp(cx, last.width);
            call(
                cx,
                WideOp::Fill,
                Some(ones),
                None,
                None,
                [u64::from(last.width), wide_rt::FILL_ONE, 0, 0],
            );
            write_levels(
                cx,
                mask,
                w,
                &levels,
                HugeValue {
                    ptr: ones,
                    width: last.width,
                    signed: false,
                },
            )?;
            release(ones);
        }
    }
    release(v.ptr);
    stage_masked_image(cx, handle, slot, None, w, image, mask, bad, delay);
    Ok(())
}

/// `m[i] <= v` (or a select of the element) on a module memory whose elements
/// are wider than 65535 bits: the written bits as a value image plus a mask
/// image, staged against row `i` through `sv_rt_nba_masked` (applied in the
/// NBA region). An index that is unknown or out of range stages nothing.
pub(crate) fn lower_nba_mem_elem<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    handle: PointerValue<'ctx>,
    lhs: sv_lang::Expression<'d>,
    rhs: LoweredValue<'ctx>,
    delay: IntValue<'ctx>,
) -> Result<(), LoweringError> {
    require_four_state(cx)?;
    let (root, levels) = resolve(cx, lhs)?;
    let (r, info) = mem_row(cx, root)?;
    let w = info.elem_width;
    let v = to_huge(
        cx,
        rhs,
        "the value assigned to a vector wider than 65535 bits must be integral",
    )?;
    let image = temp(cx, w);
    let mask = temp(cx, w);
    let fill = |p: PointerValue<'ctx>, kind: u64| {
        call(
            cx,
            WideOp::Fill,
            Some(p),
            None,
            None,
            [u64::from(w), kind, 0, 0],
        );
    };
    fill(image, wide_rt::FILL_ZERO);
    let mut bad = if r.valid.get_zero_extended_constant() == Some(1) {
        None
    } else {
        Some(cx.b.build_not(r.valid, "hmem_oob").unwrap())
    };
    match levels.last() {
        None => {
            call(
                cx,
                WideOp::Resize,
                Some(image),
                Some(v.ptr),
                None,
                [u64::from(w), v.info(), 0, 0],
            );
            fill(mask, wide_rt::FILL_ONE);
        }
        Some(last) => {
            if last.width != v.width {
                return loud(
                    cx,
                    "internal: the value assigned to a select of a huge vector is not as wide \
                     as the select",
                );
            }
            fill(mask, wide_rt::FILL_ZERO);
            for l in &levels {
                bad = or_bad(cx, bad, l.bad);
            }
            write_levels(cx, image, w, &levels, v)?;
            let ones = temp(cx, last.width);
            call(
                cx,
                WideOp::Fill,
                Some(ones),
                None,
                None,
                [u64::from(last.width), wide_rt::FILL_ONE, 0, 0],
            );
            write_levels(
                cx,
                mask,
                w,
                &levels,
                HugeValue {
                    ptr: ones,
                    width: last.width,
                    signed: false,
                },
            )?;
            release(ones);
        }
    }
    release(v.ptr);
    stage_masked_image(
        cx,
        handle,
        info.slot,
        Some(r.base),
        w,
        image,
        mask,
        bad,
        delay,
    );
    Ok(())
}

/// `obj.p <= v` (or a select of the property) on a class property wider than
/// 65535 bits: the written bits as a value image and a mask image in the
/// property's own packed words (a `PK` image is exactly the field's words),
/// staged against the object through `ObjOp::FieldNba` and merged in the NBA
/// region (LRM 1800-2023 10.4.2). A select window that is unknown or out of
/// range stages nothing; a null handle stages nothing.
pub(crate) fn lower_nba_class_prop<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    lhs: sv_lang::Expression<'d>,
    rhs: LoweredValue<'ctx>,
    delay: IntValue<'ctx>,
) -> Result<(), LoweringError> {
    require_four_state(cx)?;
    let (root, levels) = resolve(cx, lhs)?;
    let Some((h, slot, ty, dead)) = crate::classes::class_field_loc(cx, root)? else {
        return loud(cx, "internal: a class property lvalue with no location");
    };
    let w = ty.value_width();
    let v = to_huge(
        cx,
        rhs,
        "the value assigned to a vector wider than 65535 bits must be integral",
    )?;
    if let Some((name, at)) = dead {
        // Liveness of the handle: a one-word read of the property's storage.
        let probe = crate::expr::entry_alloca(cx, cx.ctx.i64_type(), "hcnba_live");
        let live = crate::classes::field_get_into(
            cx,
            h,
            vj_sim::objects::FieldSlot {
                offset: slot.offset,
                words: 1,
            },
            probe,
        );
        crate::classes::report_dead_handle_access(cx, live, at, &name, "write")?;
    }
    let image = temp(cx, w);
    let mask = temp(cx, w);
    let fill = |p: PointerValue<'ctx>, width: u32, kind: u64| {
        call(
            cx,
            WideOp::Fill,
            Some(p),
            None,
            None,
            [u64::from(width), kind, 0, 0],
        );
    };
    fill(image, w, wide_rt::FILL_ZERO);
    let mut bad: Option<IntValue<'ctx>> = None;
    match levels.last() {
        None => {
            call(
                cx,
                WideOp::Resize,
                Some(image),
                Some(v.ptr),
                None,
                [u64::from(w), v.info(), 0, 0],
            );
            // Both planes of every bit are written.
            fill(mask, w, wide_rt::FILL_Z);
        }
        Some(last) => {
            if last.width != v.width {
                return loud(
                    cx,
                    "internal: the value assigned to a select of a huge vector is not as wide \
                     as the select",
                );
            }
            fill(mask, w, wide_rt::FILL_ZERO);
            for l in &levels {
                bad = or_bad(cx, bad, l.bad);
            }
            write_levels(cx, image, w, &levels, v)?;
            let ones = temp(cx, last.width);
            fill(ones, last.width, wide_rt::FILL_Z);
            write_levels(
                cx,
                mask,
                w,
                &levels,
                HugeValue {
                    ptr: ones,
                    width: last.width,
                    signed: false,
                },
            )?;
            release(ones);
        }
    }
    release(v.ptr);
    // The `FieldNba` buffer: the value words, the mask words, the delay.
    let i64t = cx.ctx.i64_type();
    let words = slot.words as u32;
    let func = cx.b.get_insert_block().unwrap().get_parent().unwrap();
    let done_bb = cx.ctx.append_basic_block(func, "hcnba_done");
    if let Some(bad) = bad {
        let stage_bb = cx.ctx.append_basic_block(func, "hcnba_stage");
        cx.b.build_conditional_branch(bad, done_bb, stage_bb)
            .unwrap();
        cx.b.position_at_end(stage_bb);
    }
    let buf = crate::expr::entry_alloca(cx, i64t.array_type(2 * words + 1), "hcnba_buf");
    let bytes = i64t.const_int(u64::from(words) * 8, false);
    cx.b.build_memcpy(buf, 8, image, 8, bytes).unwrap();
    // SAFETY: both offsets are inside the `2 * words + 1`-word buffer.
    let (mask_ptr, delay_ptr) = unsafe {
        (
            cx.b.build_in_bounds_gep(
                i64t,
                buf,
                &[i64t.const_int(u64::from(words), false)],
                "hcnba_mask",
            )
            .unwrap(),
            cx.b.build_in_bounds_gep(
                i64t,
                buf,
                &[i64t.const_int(u64::from(2 * words), false)],
                "hcnba_delay",
            )
            .unwrap(),
        )
    };
    cx.b.build_memcpy(mask_ptr, 8, mask, 8, bytes).unwrap();
    cx.b.build_store(delay_ptr, delay).unwrap();
    crate::classes::obj_call(
        cx,
        vj_runtime::objects_rt::ObjOp::FieldNba,
        h,
        i64t.const_int(slot.offset as u64, false),
        i64t.const_int(slot.words as u64, false),
        buf,
    );
    cx.b.build_unconditional_branch(done_bb).unwrap();
    cx.b.position_at_end(done_bb);
    release(image);
    release(mask);
    Ok(())
}

/// Hands the `w`-bit value `image` under `mask` (both PK images, released here)
/// to `sv_rt_nba_masked` for `slot` (row `row` of a memory), unless `bad`
/// says the window is invalid.
#[allow(clippy::too_many_arguments)]
fn stage_masked_image<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    handle: PointerValue<'ctx>,
    slot: u32,
    row: Option<IntValue<'ctx>>,
    w: u32,
    image: PointerValue<'ctx>,
    mask: PointerValue<'ctx>,
    bad: Option<IntValue<'ctx>>,
    delay: IntValue<'ctx>,
) {
    let nwords = w.div_ceil(64);
    // An invalid window stages nothing.
    let func = cx.b.get_insert_block().unwrap().get_parent().unwrap();
    let done_bb = cx.ctx.append_basic_block(func, "hnba_done");
    if let Some(bad) = bad {
        let stage_bb = cx.ctx.append_basic_block(func, "hnba_stage");
        cx.b.build_conditional_branch(bad, done_bb, stage_bb)
            .unwrap();
        cx.b.position_at_end(stage_bb);
    }
    let val_arr = acquire(cx, nwords);
    let unk_arr = acquire(cx, nwords);
    let mask_arr = acquire(cx, nwords);
    let scratch = acquire(cx, nwords);
    call(
        cx,
        WideOp::ToValUnk,
        Some(val_arr),
        Some(image),
        Some(unk_arr),
        [u64::from(w), 0, 0, 0],
    );
    call(
        cx,
        WideOp::ToValUnk,
        Some(mask_arr),
        Some(mask),
        Some(scratch),
        [u64::from(w), 0, 0, 0],
    );
    let i32t = cx.ctx.i32_type();
    cx.b.build_call(
        crate::select::rt_nba_masked_fn(cx),
        &[
            handle.into(),
            i32t.const_int(u64::from(slot), false).into(),
            row.unwrap_or_else(|| cx.ctx.i64_type().const_zero()).into(),
            i32t.const_int(u64::from(row.is_some()), false).into(),
            val_arr.into(),
            unk_arr.into(),
            mask_arr.into(),
            i32t.const_int(u64::from(nwords), false).into(),
            i32t.const_int(u64::from(w), false).into(),
            delay.into(),
        ],
        "rt_nba_masked_call",
    )
    .unwrap();
    for p in [val_arr, unk_arr, mask_arr, scratch, image, mask] {
        release(p);
    }
    cx.b.build_unconditional_branch(done_bb).unwrap();
    cx.b.position_at_end(done_bb);
}

// ---------------------------------------------------------------------------
// Subroutine arguments, results and locals above WIDE_THRESHOLD

/// Whether a `width`-bit vector is too wide for an LLVM integer
/// (`> WIDE_THRESHOLD`): as a subroutine argument, result or local it lives
/// in memory (the `PK` image) and crosses a call as a pointer.
pub(crate) fn is_image_width(width: u32) -> bool {
    width > WIDE_THRESHOLD
}

/// Whether `ty` is an integral type that is an image per [`is_image_width`].
pub(crate) fn is_image_type(ty: &sv_lang::Type<'_>) -> bool {
    ty.is_integral() && !ty.is_floating() && is_image_width(ty.value_width())
}

/// The storage type of a `width`-bit image: `pk_words(width)` 64-bit words.
pub(crate) fn image_storage_type<'ctx>(
    ctx: &'ctx inkwell::context::Context,
    width: u32,
) -> inkwell::types::ArrayType<'ctx> {
    ctx.i64_type().array_type(pk_words(width))
}

/// Images need the four-state representation (`sv_rt_wide` is four-state only).
pub(crate) fn require_images(
    repr: ValueRepr,
    at: &sv_lang::Symbol<'_>,
) -> Result<(), LoweringError> {
    if repr != ValueRepr::FourState {
        return Err(LoweringError::unsupported_at(
            "a vector wider than 65535 bits needs the four-state value representation",
            at,
        ));
    }
    Ok(())
}

/// Sets the `width`-bit image at `ptr` to all X (a four-state variable's
/// default, LRM 6.8) or all zero (a two-state one).
pub(crate) fn init_image<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    ptr: PointerValue<'ctx>,
    width: u32,
    four_state: bool,
) -> Result<(), LoweringError> {
    require_four_state(cx)?;
    let kind = if four_state {
        wide_rt::FILL_X
    } else {
        wide_rt::FILL_ZERO
    };
    call(
        cx,
        WideOp::Fill,
        Some(ptr),
        None,
        None,
        [u64::from(width), kind, 0, 0],
    );
    Ok(())
}

/// A fresh copy of `actual` as a `width`-bit image, for an input formal of
/// that width (the callee may assign to its formal; the actual keeps its
/// value, LRM 13.5.1).
pub(crate) fn pass_by_value<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    actual: sv_lang::Expression<'d>,
    width: u32,
) -> Result<PointerValue<'ctx>, LoweringError> {
    require_four_state(cx)?;
    let v = lower_operand(cx, actual)?;
    let dst = temp(cx, width);
    call(
        cx,
        WideOp::Resize,
        Some(dst),
        Some(v.ptr),
        None,
        [u64::from(width), v.info(), 0, 0],
    );
    release(v.ptr);
    Ok(dst)
}

/// A pooled image for a `width`-bit function result (the callee fills it;
/// the consumer releases it).
pub(crate) fn result_image<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    width: u32,
) -> Result<PointerValue<'ctx>, LoweringError> {
    require_four_state(cx)?;
    Ok(temp(cx, width))
}

/// `e` evaluated and stored to the whole image variable `sym`.
pub(crate) fn store_expr_to_image<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    sym: &sv_lang::Symbol<'d>,
    e: sv_lang::Expression<'d>,
) -> Result<(), LoweringError> {
    let v = lower_expr_h(cx, e)?;
    if try_store_symbol(cx, sym, v)? {
        Ok(())
    } else {
        loud(cx, "internal: an image variable with no local slot")
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn pk_words_is_two_planes_rounded_up_to_words() {
        assert_eq!(pk_words(1), 1);
        assert_eq!(pk_words(32), 1);
        assert_eq!(pk_words(33), 2);
        assert_eq!(pk_words(70001), 2188);
        assert_eq!(pk_words(82944), 2592);
    }

    #[test]
    fn the_default_threshold_is_below_the_integer_ceiling() {
        // The knob is only set by the differential test's child processes.
        if std::env::var(THRESHOLD_ENV).is_err() {
            assert_eq!(threshold(), DEFAULT_THRESHOLD);
            assert!(!is_huge_width(DEFAULT_THRESHOLD));
            assert!(is_huge_width(DEFAULT_THRESHOLD + 1));
        }
        assert!((64..=WIDE_THRESHOLD).contains(&DEFAULT_THRESHOLD));
    }

    #[test]
    fn pool_names_survive_llvm_uniquing() {
        assert_eq!(pool_words("hugetmp_busy_12_"), Some(12));
        assert_eq!(pool_words("hugetmp_free_2188_3"), Some(2188));
        assert_eq!(pool_words("other"), None);
    }
}