vj-codegen 0.1.0

//! R-run Task 4 (CG-095/CG-096/CG-121/PR-009): the branch/CFG-construction
//! logic for `case`/`while`/`do-while`/`for`/`repeat`, extracted out of
//! `func.rs::lower_stmt` into ONE shared implementation reused by both
//! `func.rs`'s standalone function/task-declaration lowering AND
//! `process_lower.rs`'s process-segment lowering. This is the literal fix for
//! CG-095 ("suspending and non-suspending code share ONE statement-lowering
//! visitor; there is never a second 'instrumented' codegen path for the same
//! SV semantics") -- before this task, `func.rs::lower_stmt` had the only
//! working `Case`/loop arms, and they were unreachable from
//! `process_lower.rs::lower_segment_statement`, which rejected every one of
//! these kinds outright.
//!
//! Each function below builds the LLVM control-flow SHAPE only (blocks,
//! branches, the loop/case-arm bookkeeping) -- lowering a nested BODY
//! statement (a loop body, a `case` arm's statement, a `default:` statement)
//! is always delegated back to the caller's own [`Recurse`] callback, because
//! `func.rs`'s callers and `process_lower.rs`'s callers lower a LEAF
//! statement differently: `func.rs::lower_stmt` additionally handles `Return`
//! and bare user task/function calls; `process_lower.rs::
//! lower_segment_statement` additionally handles `$display`-family system
//! calls and nonblocking (`<=`) assignments via `nba_enqueue` staging, and
//! rejects a bare task/function call and `Return` outright (neither is legal
//! inside a process segment). See `tests/shared_stmt_lowering.rs` for the
//! caller-independence guarantee this buys.
//!
//! **NamedValue read/write resolution needs no caller-specific callback**,
//! despite the R-run plan's literal text asking for one: both callers already
//! resolve every `NamedValue` uniformly through `LowerCtx::locals` --
//! `process_lower.rs`'s external (module-level) signals are shadow-copied
//! into `cx.locals` ONCE PER SEGMENT by `prime_external_signals`/
//! `flush_external_signals`, never accessed per-statement, so `lower_expr`/
//! `lower_condition` (already caller-agnostic) are reused unchanged by every
//! control-flow shape below. This is a deviation from the plan's literal
//! two-callback design, recorded in the task's own deviations: the real
//! architecture already unifies this half, so a second callback would be
//! dead plumbing.
use crate::types::TypeWidth;
use inkwell::values::FunctionValue;
use sv_sim_sim::symid::SymbolUid;

use sv_lang::{CaseStatementCondition, UniquePriorityCheck};
use sv_sim_sim::process::cfg::{if_series, is_clause_if, violation_check};
use sv_sim_sim::sched::ViolationType;

use crate::error::LoweringError;
use crate::expr::{LowerCtx, LoweredValue, lower_condition, lower_expr};
use crate::ops::{OpBuilder, is_definitely_true};
use crate::types::lower_type;
use crate::value::{PackedValue, ValueRepr};

/// A caller-supplied callback for lowering one nested statement (a loop
/// body, a `case` arm's own statement, `case`'s `default:` statement) -- the
/// exact statement-dispatch entry point each caller's OWN top-level visitor
/// exposes (`func.rs::lower_stmt` / `process_lower.rs::
/// lower_segment_statement`), so a statement nested inside a shape this
/// module builds is lowered exactly the way that caller's own (non-shared)
/// statement kinds already are.
pub(crate) type Recurse<'a, 'ctx, 'd, 'm> = dyn FnMut(&mut LowerCtx<'ctx, 'd, 'm>, sv_lang::Statement<'d>) -> Result<(), LoweringError>
    + 'a;

/// The basic block the builder is currently positioned in, and that block's
/// parent function. Shared by every control-flow shape below (previously
/// duplicated once per caller before this task).
pub(crate) fn current_block_and_fn<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
) -> Result<(inkwell::basic_block::BasicBlock<'ctx>, FunctionValue<'ctx>), LoweringError> {
    let block = cx.b.get_insert_block().ok_or_else(|| {
        LoweringError::unsupported_at("statement lowered outside a basic block", &cx.fn_sym)
    })?;
    let f = block
        .get_parent()
        .ok_or_else(|| LoweringError::unsupported_at("block has no parent function", &cx.fn_sym))?;
    Ok((block, f))
}

/// Whether the block the builder is currently positioned in already ends in
/// a terminator -- checked before appending another unconditional branch
/// (LLVM's verifier rejects a second terminator in one block).
pub(crate) fn current_block_terminated(cx: &LowerCtx) -> bool {
    cx.b.get_insert_block()
        .and_then(|b| b.get_terminator())
        .is_some()
}

/// Declares a brand-new `locals` entry for a loop-introduced variable (a
/// `for` loop's own declared loop variable, or a `foreach` loop's iterator
/// variable) -- alloca + store an initial value + `cx.locals.insert`.
pub(crate) fn declare_loop_local<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    sym: &sv_lang::Symbol<'d>,
    init: Option<sv_lang::Expression<'d>>,
) -> Result<(), LoweringError> {
    let t = sym
        .value_type()
        .ok_or_else(|| LoweringError::unsupported_at("loop variable has no type", sym))?;
    let width = t.value_width();
    let signed = t.is_signed();
    let llvm_ty = if t.is_floating() {
        // A real's cell is its IEEE bits in the repr's storage word.
        let cell = if cx.repr == crate::value::ValueRepr::FourState {
            width * 2
        } else {
            width
        };
        crate::types::int_type(cx.ctx, cell).into()
    } else {
        lower_type(cx.ctx, &t, cx.repr)?.into_basic_type_enum()
    };
    let slot = crate::types::entry_alloca_in(&cx.b, llvm_ty, sym.name());
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let init_pv = match init {
        // A real loop variable's cell holds its IEEE bits (LRM 6.12), like
        // every other real local; reads go through `bits_to_real`.
        Some(e) if t.is_floating() => match lower_expr(cx, e)? {
            LoweredValue::Real(r) => crate::real::real_to_bits(cx, r, width),
            _ => {
                return Err(LoweringError::unsupported_at(
                    "real loop variable initializer must be a real value",
                    sym,
                ));
            }
        },
        Some(e) => {
            lower_expr(cx, e)?.into_packed("loop variable initializer must be integral", sym)?
        }
        None => op.pack(
            op.i_n(width).const_zero(),
            op.i_n(width).const_zero(),
            width,
            cx.repr,
            signed,
        ),
    };
    cx.b.build_store(slot, init_pv.llvm).unwrap();
    cx.locals.insert(sym.uid(), (slot, width, signed));
    Ok(())
}

/// Task 26 (CG-121) / R-run Task 4: `case`/`casex`/`casez` -- one AST kind
/// (`Case`) covering all three plus `case ... inside`, the
/// comparison flavor carried on the statement itself via `case_condition()`.
pub(crate) fn lower_case<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    // `case (type(T))` (LRM 1800-2023 6.23): every label is a type too, so
    // the taken arm is known at compile time.
    if let Some(sel) = s.expr().and_then(|x| x.type_reference_target()) {
        let taken = s.case_items().iter().find_map(|item| {
            item.expressions()
                .iter()
                .any(|l| {
                    l.type_reference_target()
                        .is_some_and(|t| t.is_matching(&sel))
                })
                .then(|| item.stmt())
        });
        return match taken.or_else(|| s.default_case()) {
            Some(arm) => recurse(cx, arm),
            None => Ok(()),
        };
    }
    if violation_check(&s).is_some() {
        let arms: Vec<sv_lang::Statement<'d>> = s.case_items().iter().map(|i| i.stmt()).collect();
        return lower_checked_arms(cx, s, &arms, s.default_case(), recurse);
    }
    let (cond_kind, sel_pv) = case_selector(cx, s)?;

    let items = s.case_items();
    let (_, f) = current_block_and_fn(cx)?;
    let check_blocks: Vec<_> = (0..items.len())
        .map(|i| cx.ctx.append_basic_block(f, &format!("case_check_{i}")))
        .collect();
    let arm_blocks: Vec<_> = (0..items.len())
        .map(|i| cx.ctx.append_basic_block(f, &format!("case_arm_{i}")))
        .collect();
    let default_block = cx.ctx.append_basic_block(f, "case_default");
    let merge_block = cx.ctx.append_basic_block(f, "case_merge");

    let first_target = check_blocks.first().copied().unwrap_or(default_block);
    cx.b.build_unconditional_branch(first_target).unwrap();

    for (i, item) in items.iter().enumerate() {
        cx.b.position_at_end(check_blocks[i]);
        let matched = case_item_matches(cx, cond_kind, sel_pv, item)?;
        let next_check = check_blocks.get(i + 1).copied().unwrap_or(default_block);
        cx.b.build_conditional_branch(matched, arm_blocks[i], next_check)
            .unwrap();
    }

    let mut merge_reachable = false;
    for (i, item) in items.iter().enumerate() {
        cx.b.position_at_end(arm_blocks[i]);
        recurse(cx, item.stmt())?;
        if !current_block_terminated(cx) {
            cx.b.build_unconditional_branch(merge_block).unwrap();
            merge_reachable = true;
        }
    }

    cx.b.position_at_end(default_block);
    if let Some(default_stmt) = s.default_case() {
        recurse(cx, default_stmt)?;
    }
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(merge_block).unwrap();
        merge_reachable = true;
    }

    cx.b.position_at_end(merge_block);
    if !merge_reachable {
        cx.b.build_unreachable().unwrap();
    }
    Ok(())
}

/// An evaluated `case` selector: an LLVM integer, or -- when the selector or
/// any item label is wider than the huge threshold -- an image in memory.
#[derive(Clone, Copy)]
pub(crate) enum CaseSel<'ctx> {
    Packed(crate::value::PackedValue<'ctx>),
    Huge(crate::huge::HugeValue<'ctx>),
    /// A real selector: each label is compared as a real (LRM 12.5 via
    /// 11.8.1: one real operand makes the comparison real).
    Real(crate::value::RealValue<'ctx>),
}

/// The comparison kind and the evaluated selector of `case` statement `s`.
pub(crate) fn case_selector<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    s: sv_lang::Statement<'d>,
) -> Result<(CaseStatementCondition, CaseSel<'ctx>), LoweringError> {
    let sel_expr = s.expr().ok_or_else(|| {
        LoweringError::unsupported_at("case missing selector expression", &cx.fn_sym)
    })?;
    let cond_kind = s
        .case_condition()
        .ok_or_else(|| LoweringError::unsupported_at("case missing case_condition", &cx.fn_sym))?;
    let huge_case = crate::huge::is_huge_expr(sel_expr)
        || s.case_items().iter().any(|i| {
            i.expressions()
                .iter()
                .any(|x| crate::huge::candidate_is_huge(*x))
        });
    if huge_case {
        return Ok((
            cond_kind,
            CaseSel::Huge(crate::huge::lower_selector(cx, sel_expr)?),
        ));
    }
    let sel_pv = match lower_expr(cx, sel_expr)? {
        LoweredValue::Real(rv) => return Ok((cond_kind, CaseSel::Real(rv))),
        other => crate::strings::packed_or_id(
            cx,
            other,
            "case selector must be an integral (non-real) value",
        )?,
    };
    Ok((cond_kind, CaseSel::Packed(sel_pv)))
}

/// Whether any label of `item` matches the selector `sel_pv` (an `i1`; an
/// ambiguous X comparison does not match).
pub(crate) fn case_item_matches<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    cond_kind: CaseStatementCondition,
    sel: CaseSel<'ctx>,
    item: &sv_lang::CaseItem<'d>,
) -> Result<inkwell::values::IntValue<'ctx>, LoweringError> {
    let label_exprs = item.expressions();
    if label_exprs.is_empty() {
        return Err(LoweringError::unsupported_at(
            "case item with no label expressions",
            &cx.fn_sym,
        ));
    }
    let mut matched: Option<inkwell::values::IntValue<'ctx>> = None;
    for label_expr in label_exprs {
        let sel_pv = match sel {
            CaseSel::Packed(pv) => pv,
            CaseSel::Real(rv) => {
                let m = real_case_label_matches(cx, cond_kind, rv, label_expr)?;
                matched = Some(match matched {
                    None => m,
                    Some(prev) => cx.b.build_or(prev, m, "case_item_or").unwrap(),
                });
                continue;
            }
            CaseSel::Huge(h) => {
                let m = crate::huge::case_label_matches(cx, cond_kind, h, label_expr)?;
                matched = Some(match matched {
                    None => m,
                    Some(prev) => cx.b.build_or(prev, m, "case_item_or").unwrap(),
                });
                continue;
            }
        };
        let hit = if cond_kind == CaseStatementCondition::Inside {
            // `case ... inside`: each label is an `inside` candidate (a
            // value matched with `==?`, or a `[lo:hi]` range).
            crate::expr::inside_candidate(cx, sel_pv, label_expr)?
        } else {
            let label_pv = crate::strings::packed_or_id(
                cx,
                lower_expr(cx, label_expr)?,
                "case item label must be an integral (non-real) value",
            )?;
            let op = OpBuilder {
                ctx: cx.ctx,
                b: &cx.b,
            };
            match cond_kind {
                CaseStatementCondition::Normal => op.caseeq4(sel_pv, label_pv),
                CaseStatementCondition::WildcardXOrZ => op.casex_match(sel_pv, label_pv),
                CaseStatementCondition::WildcardJustZ => op.wildcard_match(sel_pv, label_pv, false),
                CaseStatementCondition::Inside => unreachable!("handled above"),
            }
        };
        let m = is_definitely_true(cx.ctx, &cx.b, hit);
        matched = Some(match matched {
            None => m,
            Some(prev) => cx.b.build_or(prev, m, "case_item_or").unwrap(),
        });
    }
    Ok(matched.expect("labels non-empty, checked above"))
}

/// Whether the real selector `sel` equals `label_expr` (LRM 12.5: a plain
/// `case` compares with `==`, a real comparison when either side is real,
/// 11.8.1; an integral label is converted, 6.12.2). NaN never matches.
fn real_case_label_matches<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    cond_kind: CaseStatementCondition,
    sel: crate::value::RealValue<'ctx>,
    label_expr: sv_lang::Expression<'d>,
) -> Result<inkwell::values::IntValue<'ctx>, LoweringError> {
    if cond_kind != CaseStatementCondition::Normal {
        return Err(LoweringError::unsupported_at(
            "casex/casez/case-inside over a real selector",
            &cx.fn_sym,
        ));
    }
    let label = match lower_expr(cx, label_expr)? {
        LoweredValue::Real(rv) => rv,
        LoweredValue::Packed(pv) => crate::real::packed_to_real(cx, pv).ok_or_else(|| {
            LoweringError::unsupported_at(
                "case item label wider than 64 bits against a real selector",
                &cx.fn_sym,
            )
        })?,
        _ => {
            return Err(LoweringError::unsupported_at(
                "case item label of a non-numeric type against a real selector",
                &cx.fn_sym,
            ));
        }
    };
    let (a, b) = (
        crate::real::as_f64(cx, sel.llvm),
        crate::real::as_f64(cx, label.llvm),
    );
    Ok(cx
        .b
        .build_float_compare(inkwell::FloatPredicate::OEQ, a, b, "real_case_eq")
        .unwrap())
}

/// A `unique`/`unique0`/`priority` `if` series or `case` lowered in place
/// (§12.4.2, §12.5.3): the dispatch with its violation checks
/// ([`lower_checked_dispatch`]), then the arm statement of the first match
/// (`arms[i]` for the i-th condition or case item) or `otherwise`, each
/// through `recurse`.
fn lower_checked_arms<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    arms: &[sv_lang::Statement<'d>],
    otherwise: Option<sv_lang::Statement<'d>>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let (_, f) = current_block_and_fn(cx)?;
    let arm_blocks: Vec<_> = (0..arms.len())
        .map(|i| cx.ctx.append_basic_block(f, &format!("checked_arm_{i}")))
        .collect();
    let default_block = cx.ctx.append_basic_block(f, "checked_default");
    let merge_block = cx.ctx.append_basic_block(f, "checked_merge");
    if violation_check(&s).is_some() {
        lower_checked_dispatch(cx, s, &arm_blocks, default_block)?;
    } else {
        lower_clause_dispatch(cx, s, &arm_blocks, default_block)?;
    }

    let mut merge_reachable = false;
    for (arm, block) in arms.iter().zip(&arm_blocks) {
        cx.b.position_at_end(*block);
        recurse(cx, *arm)?;
        if !current_block_terminated(cx) {
            cx.b.build_unconditional_branch(merge_block).unwrap();
            merge_reachable = true;
        }
    }
    cx.b.position_at_end(default_block);
    if let Some(stmt) = otherwise {
        recurse(cx, stmt)?;
    }
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(merge_block).unwrap();
        merge_reachable = true;
    }
    cx.b.position_at_end(merge_block);
    if !merge_reachable {
        cx.b.build_unreachable().unwrap();
    }
    Ok(())
}

/// The error for an `if` series [`if_series`] cannot take apart.
fn checked_if_error(why: &str, cx: &LowerCtx) -> LoweringError {
    LoweringError::unsupported_at(&format!("{why} in an if series"), &cx.fn_sym)
}

/// The match tests of a checked statement's arms, in source order.
enum ArmTests<'ctx, 'd> {
    /// The i-th condition of a checked `if` series.
    Conditions(Vec<sv_lang::Expression<'d>>),
    /// Arm `i` is taken when every clause of `arms[i]` holds (an `if` link
    /// with `&&&` or a pattern, a `case ... matches` item): clauses run left
    /// to right with short-circuit, binding pattern variables as they match.
    Clauses {
        mode: crate::tagged::MatchMode,
        arms: Vec<Vec<crate::tagged::Clause<'ctx, 'd>>>,
    },
    /// The items of a checked `case`, against its selector (evaluated once,
    /// before any item).
    Items {
        kind: CaseStatementCondition,
        selector: CaseSel<'ctx>,
        items: Vec<sv_lang::CaseItem<'d>>,
    },
}

impl<'ctx, 'd> ArmTests<'ctx, 'd> {
    fn len(&self) -> usize {
        match self {
            ArmTests::Conditions(conds) => conds.len(),
            ArmTests::Clauses { arms, .. } => arms.len(),
            ArmTests::Items { items, .. } => items.len(),
        }
    }

    /// The `i1` "arm `i` matches" test (an unknown comparison does not match).
    fn test(
        &self,
        cx: &LowerCtx<'ctx, 'd, '_>,
        i: usize,
    ) -> Result<inkwell::values::IntValue<'ctx>, LoweringError> {
        match self {
            ArmTests::Conditions(conds) => lower_condition(cx, conds[i]),
            ArmTests::Clauses { mode, arms } => crate::tagged::lower_clauses(cx, &arms[i], *mode),
            ArmTests::Items {
                kind,
                selector,
                items,
            } => case_item_matches(cx, *kind, *selector, &items[i]),
        }
    }
}

/// The arm tests of the `if` series / `case` / `case ... matches` `s` (its
/// subject evaluated here, once), whether it has an `else`/`default`, and the
/// construct and match wording of its violation reports.
fn dispatch_tests<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    s: sv_lang::Statement<'d>,
) -> Result<(ArmTests<'ctx, 'd>, bool, &'static str, &'static str), LoweringError> {
    use sv_lang::kinds::StatementKind;
    match s.kind() {
        StatementKind::Conditional => {
            let chain = if_series(s).map_err(|why| checked_if_error(why, cx))?;
            let tests = if chain.links.iter().any(is_clause_if) {
                ArmTests::Clauses {
                    mode: crate::tagged::MatchMode::Exact,
                    arms: crate::tagged::if_arms(cx, &chain.links)?,
                }
            } else {
                ArmTests::Conditions(
                    chain
                        .links
                        .iter()
                        .map(|l| l.conditions()[0].expr())
                        .collect(),
                )
            };
            Ok((tests, chain.otherwise.is_some(), "if", "condition is true"))
        }
        StatementKind::PatternCase => {
            let (mode, arms) = crate::tagged::case_arms(cx, s)?;
            let construct = match mode {
                crate::tagged::MatchMode::Exact => "case matches",
                crate::tagged::MatchMode::Z => "casez matches",
                crate::tagged::MatchMode::X => "casex matches",
            };
            Ok((
                ArmTests::Clauses { mode, arms },
                s.pattern_case_default().is_some(),
                construct,
                "case item matches the case expression",
            ))
        }
        StatementKind::Case => {
            let (kind, selector) = case_selector(cx, s)?;
            let construct = match kind {
                CaseStatementCondition::Normal => "case",
                CaseStatementCondition::WildcardXOrZ => "casex",
                CaseStatementCondition::WildcardJustZ => "casez",
                CaseStatementCondition::Inside => "case inside",
            };
            Ok((
                ArmTests::Items {
                    kind,
                    selector,
                    items: s.case_items(),
                },
                s.default_case().is_some(),
                construct,
                "case item matches the case expression",
            ))
        }
        other => Err(LoweringError::unsupported_at(
            &format!(
                "a {other:?} statement has no arm tests (only `if`, `case` and `case ... \
                 matches` do)"
            ),
            &cx.fn_sym,
        )),
    }
}

/// Emits the dispatch of an unqualified `if` with a pattern-matching or
/// `&&&` predicate, or `case ... matches`, at the current position, ending the
/// current block: control goes to `arms[i]` for the first arm whose clauses
/// all hold (pattern variables bound on the way), else to `default`.
pub(crate) fn lower_clause_dispatch<'ctx, 'd, 'm>(
    cx: &LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    arms: &[inkwell::basic_block::BasicBlock<'ctx>],
    default: inkwell::basic_block::BasicBlock<'ctx>,
) -> Result<(), LoweringError> {
    let (tests, _, _, _) = dispatch_tests(cx, s)?;
    if tests.len() != arms.len() {
        return Err(LoweringError::unsupported_at(
            "a pattern dispatch whose arm count differs from its tests (internal invariant)",
            &cx.fn_sym,
        ));
    }
    let (_, f) = current_block_and_fn(cx)?;
    for (i, arm) in arms.iter().enumerate() {
        let m = tests.test(cx, i)?;
        let next = cx.ctx.append_basic_block(f, "pattern_next");
        cx.b.build_conditional_branch(m, *arm, next).unwrap();
        cx.b.position_at_end(next);
    }
    cx.b.build_unconditional_branch(default).unwrap();
    Ok(())
}

/// `case (e) matches ...` (LRM 12.6.1) lowered in place, each arm through
/// `recurse`.
pub(crate) fn lower_pattern_case<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let arms: Vec<sv_lang::Statement<'d>> =
        s.pattern_case_items().iter().map(|i| i.stmt()).collect();
    lower_checked_arms(cx, s, &arms, s.pattern_case_default(), recurse)
}

/// Emits the dispatch of a `unique`/`unique0`/`priority` `if` series or `case`
/// `s` at the current position, ending the current block: control goes to
/// `arms[i]` for the first matching arm, else to `default`, and a violation
/// the statement defines is reported (§12.4.2, §12.5.3) -- held until the
/// Observed region by the runtime (§12.4.2.1, [`emit_violation_report`]).
///
/// * `priority` tests the arms in order, stopping at the first match; no match
///   and no `else`/`default` is a violation.
/// * `unique`/`unique0` test EVERY arm (the evaluations continue after a true
///   one) and violate when more than one matches; `unique` also violates when
///   none does and there is no `else`/`default`. The first matching arm runs.
pub(crate) fn lower_checked_dispatch<'ctx, 'd, 'm>(
    cx: &LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    arms: &[inkwell::basic_block::BasicBlock<'ctx>],
    default: inkwell::basic_block::BasicBlock<'ctx>,
) -> Result<(), LoweringError> {
    let check = violation_check(&s).ok_or_else(|| {
        LoweringError::unsupported_at("a checked dispatch of an unchecked statement", &cx.fn_sym)
    })?;
    let at = s.source_range().start;
    let (tests, has_default, construct, unit) = dispatch_tests(cx, s)?;
    if tests.len() != arms.len() {
        return Err(LoweringError::unsupported_at(
            "a checked dispatch whose arm count differs from its tests (internal invariant)",
            &cx.fn_sym,
        ));
    }
    let (qualifier, ty) = match check {
        UniquePriorityCheck::Unique => ("unique", ViolationType::Unique),
        UniquePriorityCheck::Unique0 => ("unique0", ViolationType::Unique0),
        UniquePriorityCheck::Priority => ("priority", ViolationType::Priority),
        UniquePriorityCheck::None => {
            return Err(LoweringError::unsupported_at(
                "a checked dispatch of an unqualified statement (internal invariant)",
                &cx.fn_sym,
            ));
        }
    };
    let none_text = format!("{qualifier} {construct} violation: no {unit}");
    let multi_text = format!("{qualifier} {construct} violation: more than one {unit}");
    let (_, f) = current_block_and_fn(cx)?;
    let i1 = cx.ctx.bool_type();

    if check == UniquePriorityCheck::Priority {
        for (i, arm) in arms.iter().enumerate() {
            let m = tests.test(cx, i)?;
            let next = cx.ctx.append_basic_block(f, "priority_next");
            cx.b.build_conditional_branch(m, *arm, next).unwrap();
            cx.b.position_at_end(next);
        }
        if !has_default {
            emit_violation_report(cx, ty, at, &none_text)?;
        }
        cx.b.build_unconditional_branch(default).unwrap();
        return Ok(());
    }

    // `unique`/`unique0`: every arm is tested up front.
    let mut matches = Vec::with_capacity(arms.len());
    for i in 0..arms.len() {
        matches.push(tests.test(cx, i)?);
    }
    let mut seen = i1.const_zero();
    let mut overlap = i1.const_zero();
    for m in &matches {
        let both = cx.b.build_and(seen, *m, "unique_both").unwrap();
        overlap = cx.b.build_or(overlap, both, "unique_overlap").unwrap();
        seen = cx.b.build_or(seen, *m, "unique_seen").unwrap();
    }
    let report_multi = cx.ctx.append_basic_block(f, "unique_multi");
    let after_multi = cx.ctx.append_basic_block(f, "unique_after_multi");
    cx.b.build_conditional_branch(overlap, report_multi, after_multi)
        .unwrap();
    cx.b.position_at_end(report_multi);
    emit_violation_report(cx, ty, at, &multi_text)?;
    cx.b.build_unconditional_branch(after_multi).unwrap();
    cx.b.position_at_end(after_multi);
    if check == UniquePriorityCheck::Unique && !has_default {
        let none = cx.b.build_not(seen, "unique_none").unwrap();
        let report_none = cx.ctx.append_basic_block(f, "unique_none");
        let after_none = cx.ctx.append_basic_block(f, "unique_after_none");
        cx.b.build_conditional_branch(none, report_none, after_none)
            .unwrap();
        cx.b.position_at_end(report_none);
        emit_violation_report(cx, ty, at, &none_text)?;
        cx.b.build_unconditional_branch(after_none).unwrap();
        cx.b.position_at_end(after_none);
    }
    for (m, arm) in matches.iter().zip(arms) {
        let next = cx.ctx.append_basic_block(f, "unique_next");
        cx.b.build_conditional_branch(*m, *arm, next).unwrap();
        cx.b.position_at_end(next);
    }
    cx.b.build_unconditional_branch(default).unwrap();
    Ok(())
}

/// Prints one violation report (`WARNING: file:line: <text>`, then the time
/// and scope) of a checked statement at `at` and hands it to the runtime as a
/// `ty` violation report of the running process, which holds it until the
/// Observed region (`sv_sim_sim::sched::violation_report`).
fn emit_violation_report<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    ty: ViolationType,
    at: sv_lang::SourceLoc,
    text: &str,
) -> Result<(), LoweringError> {
    let start = crate::process_lower::deferred_begin_call(cx);
    let sys_fns = crate::process_lower::fetch_or_declare_sys_call_fns(cx.ctx, cx.module);
    let scope = sv_sim_sim::process::cfg::shown_path(&cx.fn_sym);
    let text = crate::process_lower::build_c_string_global(
        cx.ctx,
        cx.module,
        &text.replace('%', "%%"),
        "violation_msg",
    );
    let write = sys_fns.write;
    crate::process_lower::severity_frame(cx, &sys_fns, "WARNING: ", at, &scope, &mut || {
        let rtarg_ty = crate::systask::rtarg_llvm_type(cx.ctx);
        let (args_ptr, n) = crate::systask::build_rtarg_array(cx, rtarg_ty, Vec::new())?;
        cx.b.build_call(
            write,
            &[text.into(), args_ptr.into(), n.into()],
            "violation_text",
        )
        .unwrap();
        Ok(())
    })?;
    // 20.11: the control site of the report, which `$assertoff`/`$assertkill` of
    // the violation report types act on.
    let site = crate::assert_site::violation_site(&cx.fn_sym, ty.assertion_type_bit()).encode();
    crate::process_lower::deferred_end_call(cx, start, 0, ty.deferred_mode(), Some(&site));
    Ok(())
}

/// Task 26 (CG-121) / R-run Task 4: `while` -- header block (re-evaluated
/// every iteration) branches into the body or straight past it.
pub(crate) fn lower_while_loop<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let (_, f) = current_block_and_fn(cx)?;
    let header = cx.ctx.append_basic_block(f, "while_header");
    let body_block = cx.ctx.append_basic_block(f, "while_body");
    let exit_block = cx.ctx.append_basic_block(f, "while_exit");
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(header);
    // VPI `cbStmt` (Table 38-6): before the condition, every iteration.
    crate::vpi_dyn::emit_stmt_probe(cx, &s);
    let cond_expr = s
        .condition()
        .ok_or_else(|| LoweringError::unsupported_at("while missing condition", &cx.fn_sym))?;
    let cond_true = lower_condition(cx, cond_expr)?;
    cx.b.build_conditional_branch(cond_true, body_block, exit_block)
        .unwrap();

    cx.b.position_at_end(body_block);
    let body_stmt = s
        .body()
        .ok_or_else(|| LoweringError::unsupported_at("while missing body", &cx.fn_sym))?;
    lower_loop_body(cx, body_stmt, header, exit_block, recurse)?;
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(header).unwrap();
    }

    cx.b.position_at_end(exit_block);
    Ok(())
}

/// Task 26 (CG-121) / R-run Task 4: `do`/`while` -- the body runs
/// unconditionally once, THEN the condition gates every subsequent
/// iteration -- the one structural difference from `while` above.
pub(crate) fn lower_do_while_loop<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let (_, f) = current_block_and_fn(cx)?;
    let body_block = cx.ctx.append_basic_block(f, "dowhile_body");
    let footer_block = cx.ctx.append_basic_block(f, "dowhile_footer");
    let exit_block = cx.ctx.append_basic_block(f, "dowhile_exit");
    cx.b.build_unconditional_branch(body_block).unwrap();

    cx.b.position_at_end(body_block);
    let body_stmt = s
        .body()
        .ok_or_else(|| LoweringError::unsupported_at("do-while missing body", &cx.fn_sym))?;
    lower_loop_body(cx, body_stmt, footer_block, exit_block, recurse)?;
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(footer_block).unwrap();
    }

    // The footer is reached by falling off the body or by `continue`; a
    // footer or exit with no predecessor is harmless dead code.
    cx.b.position_at_end(footer_block);
    let cond_expr = s
        .condition()
        .ok_or_else(|| LoweringError::unsupported_at("do-while missing condition", &cx.fn_sym))?;
    let cond_true = lower_condition(cx, cond_expr)?;
    cx.b.build_conditional_branch(cond_true, body_block, exit_block)
        .unwrap();

    cx.b.position_at_end(exit_block);
    Ok(())
}

/// Task 26 (CG-121) / R-run Task 4: `for` -- initializer(s) run once before
/// the header; the header evaluates the stop expression every iteration; the
/// step expression(s) run at the end of every iteration, before re-testing
/// the header.
pub(crate) fn lower_for_loop<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    // VPI `cbStmt` (Table 38-6): before any of the control expressions ...
    crate::vpi_dyn::emit_stmt_probe(cx, &s);
    for init_expr in s.for_loop_initializers() {
        crate::huge::discard(lower_expr(cx, init_expr)?);
    }
    for var_sym in s.for_loop_vars() {
        let init = var_sym.initializer();
        declare_loop_local(cx, &var_sym, init)?;
    }

    let (_, f) = current_block_and_fn(cx)?;
    let header = cx.ctx.append_basic_block(f, "for_header");
    let body_block = cx.ctx.append_basic_block(f, "for_body");
    let step_block = cx.ctx.append_basic_block(f, "for_step");
    let exit_block = cx.ctx.append_basic_block(f, "for_exit");
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(header);
    // `for (;;)` has no stop expression: it runs until a `break`/`return`.
    match s.condition() {
        Some(cond_expr) => {
            let cond_true = lower_condition(cx, cond_expr)?;
            cx.b.build_conditional_branch(cond_true, body_block, exit_block)
                .unwrap();
        }
        None => {
            cx.b.build_unconditional_branch(body_block).unwrap();
        }
    }

    cx.b.position_at_end(body_block);
    let body_stmt = s
        .body()
        .ok_or_else(|| LoweringError::unsupported_at("for missing body", &cx.fn_sym))?;
    lower_loop_body(cx, body_stmt, step_block, exit_block, recurse)?;
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(step_block).unwrap();
    }

    // The step is reached by falling off the body or by `continue`.
    cx.b.position_at_end(step_block);
    // ... and every iteration, before the incremental statement.
    crate::vpi_dyn::emit_stmt_probe(cx, &s);
    for step_expr in s.for_loop_steps() {
        crate::huge::discard(lower_expr(cx, step_expr)?);
    }
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(exit_block);
    Ok(())
}

/// Task 26 (CG-121) / R-run Task 4: `repeat` -- `condition()` doubles as the
/// per-iteration COUNT expression for this statement kind, evaluated ONCE
/// before the loop starts.
pub(crate) fn lower_repeat_loop<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let count_expr = s.condition().ok_or_else(|| {
        LoweringError::unsupported_at("repeat missing count expression", &cx.fn_sym)
    })?;
    let count_pv = match lower_expr(cx, count_expr)? {
        LoweredValue::Real(rv) => crate::real::real_to_count(cx, rv),
        other => other.into_packed(
            "repeat count must be an integral (non-real) value",
            &cx.fn_sym,
        )?,
    };
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let width = count_pv.width;
    let val = op.lane_val(count_pv);
    let unk = op.lane_unk(count_pv);
    let zero_c = op.i_n(width).const_zero();
    let unk_zero =
        cx.b.build_int_compare(inkwell::IntPredicate::EQ, unk, zero_c, "repeat_unk_zero")
            .unwrap();
    let n0 =
        cx.b.build_select(unk_zero, val, zero_c, "repeat_n0")
            .unwrap()
            .into_int_value();
    let n = if count_pv.signed {
        let is_neg =
            cx.b.build_int_compare(inkwell::IntPredicate::SLT, n0, zero_c, "repeat_is_neg")
                .unwrap();
        cx.b.build_select(is_neg, zero_c, n0, "repeat_n")
            .unwrap()
            .into_int_value()
    } else {
        n0
    };

    let counter_slot = crate::types::entry_alloca_in(&cx.b, op.i_n(width), "repeat_counter");
    cx.b.build_store(counter_slot, n).unwrap();

    let (_, f) = current_block_and_fn(cx)?;
    let header = cx.ctx.append_basic_block(f, "repeat_header");
    let body_block = cx.ctx.append_basic_block(f, "repeat_body");
    let exit_block = cx.ctx.append_basic_block(f, "repeat_exit");
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(header);
    // VPI `cbStmt` (Table 38-6): when first encountered and at every
    // iteration.
    crate::vpi_dyn::emit_stmt_probe(cx, &s);
    let remaining =
        cx.b.build_load(op.i_n(width), counter_slot, "repeat_remaining")
            .unwrap()
            .into_int_value();
    let cont =
        cx.b.build_int_compare(
            inkwell::IntPredicate::NE,
            remaining,
            op.i_n(width).const_zero(),
            "repeat_cont",
        )
        .unwrap();
    cx.b.build_conditional_branch(cont, body_block, exit_block)
        .unwrap();

    // The latch counts the iteration down (reached by falling off the body
    // or by `continue`).
    let latch = cx.ctx.append_basic_block(f, "repeat_latch");
    cx.b.position_at_end(body_block);
    let body_stmt = s
        .body()
        .ok_or_else(|| LoweringError::unsupported_at("repeat missing body", &cx.fn_sym))?;
    lower_loop_body(cx, body_stmt, latch, exit_block, recurse)?;
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(latch).unwrap();
    }
    cx.b.position_at_end(latch);
    let op2 = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let remaining2 =
        cx.b.build_load(op2.i_n(width), counter_slot, "repeat_remaining2")
            .unwrap()
            .into_int_value();
    let one = op2.i_n(width).const_int(1, false);
    let dec = cx.b.build_int_sub(remaining2, one, "repeat_dec").unwrap();
    cx.b.build_store(counter_slot, dec).unwrap();
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(exit_block);
    Ok(())
}

/// Lowers a loop body with `continue` going to `cont` and `break` to `brk`.
pub(crate) fn lower_loop_body<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    body: sv_lang::Statement<'d>,
    cont: inkwell::basic_block::BasicBlock<'ctx>,
    brk: inkwell::basic_block::BasicBlock<'ctx>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    cx.loop_exits.push((cont, brk));
    let r = recurse(cx, body);
    cx.loop_exits.pop();
    r
}

/// `break;` / `continue;`: a jump to the innermost loop's exit / next
/// iteration. The current block is left terminated, so the statements
/// after it (dead code) are skipped like those after a `return`.
pub(crate) fn lower_loop_jump(
    cx: &LowerCtx<'_, '_, '_>,
    is_break: bool,
) -> Result<(), LoweringError> {
    let &(cont, brk) = cx.loop_exits.last().ok_or_else(|| {
        LoweringError::unsupported_at(
            "break/continue with no enclosing loop in this lowering",
            &cx.fn_sym,
        )
    })?;
    cx.b.build_unconditional_branch(if is_break { brk } else { cont })
        .unwrap();
    Ok(())
}

/// `foreach (a[i, j, ...])`: one loop per named dimension, outermost first,
/// each index walking from the left bound to the right bound (LRM
/// 1800-2023 §12.7.3); a dimension whose index is omitted (`foreach (a[i,
/// ])`) is not iterated. The nest is one loop to `break`/`continue`
/// (Verilator agrees): `continue` moves to the next element, `break` leaves
/// the whole nest.
pub(crate) fn lower_foreach_loop<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    if let Some(arr) = s.expressions().first().copied()
        && arr.expr_type().is_some_and(|t| t.is_string())
    {
        return lower_string_foreach(cx, s, arr, recurse);
    }
    if let Some(arr) = s.expressions().first().copied()
        && crate::collections::coll_of_expr(&arr).is_some()
    {
        return lower_collection_foreach(cx, s, arr, recurse);
    }
    if let Some(arr) = s.expressions().first().copied()
        && let Some(shape) = arr.expr_type().and_then(|t| crate::arrays::array_shape(&t))
        && let Some(child) = crate::collections::coll_of(&shape.elem)
        && s.foreach_loop_dims().len() > shape.dims.len()
    {
        return lower_fixed_outer_foreach(cx, s, arr, &shape.dims, Some(child), recurse);
    }
    // `string f[2]` with `foreach (f[i, j])`: `j` walks the bytes of `f[i]`.
    if let Some(arr) = s.expressions().first().copied()
        && let Some(shape) = arr.expr_type().and_then(|t| crate::arrays::array_shape(&t))
        && shape.elem.is_string()
        && s.foreach_loop_dims().len() == shape.dims.len() + 1
    {
        return lower_fixed_outer_foreach(cx, s, arr, &shape.dims, None, recurse);
    }
    let mut loops = Vec::new();
    for dim in s.foreach_loop_dims() {
        let Some(var) = dim.loop_var() else {
            continue;
        };
        let range = dim.range().ok_or_else(|| {
            LoweringError::unsupported_at(
                "foreach over a dynamically-sized array (dynamic array, queue or \
                 associative array)",
                &cx.fn_sym,
            )
        })?;
        loops.push((var, range.left, range.right));
    }
    // Every index omitted (`foreach (a[])`, `foreach (a[,,])`): no dimension
    // is iterated, so the body never runs (as for the collection path).
    if loops.is_empty() {
        return Ok(());
    }
    let body_stmt = s
        .body()
        .ok_or_else(|| LoweringError::unsupported_at("foreach missing body", &cx.fn_sym))?;
    let (_, f) = current_block_and_fn(cx)?;
    let exit = cx.ctx.append_basic_block(f, "foreach_break");
    lower_foreach_nest(cx, &loops, body_stmt, exit, recurse)?;
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(exit).unwrap();
    }
    cx.b.position_at_end(exit);
    Ok(())
}

/// `foreach (s[i])` over a `string`: `i` walks the byte positions 0 ..
/// len-1 (LRM 1800-2023 §6.16, §12.7.3). The length is taken once, at loop
/// entry. A string has one dimension, so any further index is refused.
fn lower_string_foreach<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    arr: sv_lang::Expression<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let dims = s.foreach_loop_dims();
    if dims.len() != 1 {
        return Err(LoweringError::unsupported_at(
            "foreach over a string with more than one index",
            &cx.fn_sym,
        ));
    }
    let Some(var) = dims[0].loop_var() else {
        // `foreach (s[])`: no dimension is iterated.
        return Ok(());
    };
    let body_stmt = s
        .body()
        .ok_or_else(|| LoweringError::unsupported_at("foreach missing body", &cx.fn_sym))?;
    let size = crate::strings::len_of(cx, arr)?;
    lower_string_bytes_loop(cx, &var, size, body_stmt, None, recurse)
}

/// The loop `var` over byte positions 0 .. `size`-1 of a string around
/// `body_stmt`; `brk` is where `break` goes (the loop's own exit by default).
fn lower_string_bytes_loop<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    var: &sv_lang::Symbol<'d>,
    size: inkwell::values::IntValue<'ctx>,
    body_stmt: sv_lang::Statement<'d>,
    brk: Option<inkwell::basic_block::BasicBlock<'ctx>>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    declare_loop_local(cx, var, None)?;
    let (slot, width, signed) = *cx.locals.get(&var.uid()).unwrap();
    let i64t = cx.ctx.i64_type();
    let pos = crate::expr::entry_alloca(cx, i64t, "sforeach_pos");
    cx.b.build_store(pos, i64t.const_zero()).unwrap();

    let (_, f) = current_block_and_fn(cx)?;
    let header = cx.ctx.append_basic_block(f, "sforeach_header");
    let body_block = cx.ctx.append_basic_block(f, "sforeach_body");
    let step_block = cx.ctx.append_basic_block(f, "sforeach_step");
    let exit_block = cx.ctx.append_basic_block(f, "sforeach_exit");
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(header);
    let p =
        cx.b.build_load(i64t, pos, "sforeach_p")
            .unwrap()
            .into_int_value();
    let more =
        cx.b.build_int_compare(inkwell::IntPredicate::SLT, p, size, "sforeach_more")
            .unwrap();
    cx.b.build_conditional_branch(more, body_block, exit_block)
        .unwrap();

    cx.b.position_at_end(body_block);
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let ty = op.i_n(width);
    let i = if width > 64 {
        cx.b.build_int_z_extend(p, ty, "sforeach_i").unwrap()
    } else {
        cx.b.build_int_truncate_or_bit_cast(p, ty, "sforeach_i")
            .unwrap()
    };
    let v = op.pack(i, ty.const_zero(), width, cx.repr, signed);
    cx.b.build_store(slot, v.llvm).unwrap();
    lower_loop_body(
        cx,
        body_stmt,
        step_block,
        brk.unwrap_or(exit_block),
        recurse,
    )?;
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(step_block).unwrap();
    }

    cx.b.position_at_end(step_block);
    let p =
        cx.b.build_load(i64t, pos, "sforeach_p2")
            .unwrap()
            .into_int_value();
    let next =
        cx.b.build_int_add(p, i64t.const_int(1, false), "sforeach_next")
            .unwrap();
    cx.b.build_store(pos, next).unwrap();
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(exit_block);
    Ok(())
}

/// `foreach (c[i])` over a queue or dynamic array (`i` walks 0 .. size-1)
/// or an associative array (`i` takes each key in order, §12.7.3). The size
/// is taken once, at loop entry.
fn lower_collection_foreach<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    arr: sv_lang::Expression<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let dims: Vec<ForeachDim<'d>> = s
        .foreach_loop_dims()
        .iter()
        .map(|d| (d.loop_var(), d.range().map(|r| (r.left, r.right))))
        .collect();
    let body_stmt = s
        .body()
        .ok_or_else(|| LoweringError::unsupported_at("foreach missing body", &cx.fn_sym))?;
    let c = crate::collections::coll_of_expr(&arr).ok_or_else(|| {
        LoweringError::unsupported_at("foreach over a non-collection", &cx.fn_sym)
    })?;
    let h = crate::collections::handle_of(cx, arr)?;
    foreach_coll_level(cx, &dims, &c, h, body_stmt, None, recurse)
}

/// `foreach (a[i, j, ...])` over a fixed-size unpacked array (one or more
/// fixed dimensions: `int a[3][]`, `int a[2][3][]`) whose elements are
/// queues or dynamic arrays (LRM 1800-2023 §7.4.5, §12.7.3): each fixed
/// index walks its bounds `(left, right)`, and for each combination the
/// handle stored in that element is the collection the remaining
/// dimensions walk ([`foreach_coll_level`]).
fn lower_fixed_outer_foreach<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    arr: sv_lang::Expression<'d>,
    fixed: &[(i64, i64)],
    child: Option<crate::collections::Coll<'d>>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let dims: Vec<ForeachDim<'d>> = s
        .foreach_loop_dims()
        .iter()
        .map(|d| (d.loop_var(), d.range().map(|r| (r.left, r.right))))
        .collect();
    let body_stmt = s
        .body()
        .ok_or_else(|| LoweringError::unsupported_at("foreach missing body", &cx.fn_sym))?;
    let mut vars = Vec::new();
    for d in &dims[..fixed.len()] {
        let Some(var) = d.0 else {
            return Err(LoweringError::unsupported_at(
                "foreach with an omitted index in front of a queue or dynamic-array \
                 dimension nested inside a fixed-size array",
                &cx.fn_sym,
            ));
        };
        vars.push(var);
    }
    let src = match crate::arrays::resolve_array_ref(cx, arr)? {
        Some((r, _)) => RowSrc::Stored(r.backing, r.base),
        // A class property or struct field: the whole array is one packed
        // value of 64-bit handle slots, lowest row first.
        None => {
            let whole = crate::expr::lower_expr(cx, arr)?
                .into_packed("a fixed array of queues or dynamic arrays", &cx.fn_sym)?;
            RowSrc::Packed(
                OpBuilder {
                    ctx: cx.ctx,
                    b: &cx.b,
                }
                .lane_val(whole),
            )
        }
    };
    let (_, f) = current_block_and_fn(cx)?;
    let exit_block = cx.ctx.append_basic_block(f, "fforeach_exit");
    let acc = cx.ctx.i64_type().const_zero();
    fixed_outer_level(
        cx,
        &vars,
        fixed,
        &dims[fixed.len()..],
        src,
        acc,
        child.as_ref(),
        body_stmt,
        exit_block,
        recurse,
    )?;
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(exit_block).unwrap();
    }
    cx.b.position_at_end(exit_block);
    Ok(())
}

/// Where the element handles of a fixed-size array of collections live.
#[derive(Clone, Copy)]
enum RowSrc<'ctx> {
    /// A local array or module memory (first row `base`).
    Stored(
        crate::arrays::Backing<'ctx>,
        inkwell::values::IntValue<'ctx>,
    ),
    /// A class property or struct field, read as one wide value.
    Packed(inkwell::values::IntValue<'ctx>),
}

/// One fixed dimension of [`lower_fixed_outer_foreach`]: `vars[0]` walks
/// `fixed[0]`, `acc` is the row-major row offset of the outer indices, and
/// the innermost level loads the element handle and hands `rest` to
/// [`foreach_coll_level`]. `brk` is the exit of the whole nest.
#[allow(clippy::too_many_arguments)]
fn fixed_outer_level<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    vars: &[sv_lang::Symbol<'d>],
    fixed: &[(i64, i64)],
    rest: &[ForeachDim<'d>],
    src: RowSrc<'ctx>,
    acc: inkwell::values::IntValue<'ctx>,
    child: Option<&crate::collections::Coll<'d>>,
    body_stmt: sv_lang::Statement<'d>,
    brk: inkwell::basic_block::BasicBlock<'ctx>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let var = vars[0];
    let (left, right) = fixed[0];
    declare_loop_local(cx, &var, None)?;
    let (slot, width, signed) = *cx.locals.get(&var.uid()).unwrap();
    let count = (left - right).unsigned_abs() + 1;
    let ascending = left <= right;
    let i64t = cx.ctx.i64_type();
    let pos = crate::expr::entry_alloca(cx, i64t, "foreach_pos");
    cx.b.build_store(pos, i64t.const_zero()).unwrap();

    let (_, f) = current_block_and_fn(cx)?;
    let header = cx.ctx.append_basic_block(f, "fforeach_header");
    let body_block = cx.ctx.append_basic_block(f, "fforeach_body");
    let step_block = cx.ctx.append_basic_block(f, "fforeach_step");
    let level_exit = cx.ctx.append_basic_block(f, "fforeach_level_exit");
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(header);
    let p =
        cx.b.build_load(i64t, pos, "fforeach_p")
            .unwrap()
            .into_int_value();
    let more =
        cx.b.build_int_compare(
            inkwell::IntPredicate::ULT,
            p,
            i64t.const_int(count, false),
            "fforeach_more",
        )
        .unwrap();
    cx.b.build_conditional_branch(more, body_block, level_exit)
        .unwrap();

    cx.b.position_at_end(body_block);
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    // The index value `left +- p`, and the stored row (the index less the
    // lowest bound).
    let lc = i64t.const_int(left as u64, true);
    let (idx, row_off) = if ascending {
        (cx.b.build_int_add(lc, p, "fforeach_idx").unwrap(), p)
    } else {
        (
            cx.b.build_int_sub(lc, p, "fforeach_idx").unwrap(),
            cx.b.build_int_sub(i64t.const_int(count - 1, false), p, "fforeach_row")
                .unwrap(),
        )
    };
    let ty = op.i_n(width);
    let iv =
        cx.b.build_int_truncate_or_bit_cast(idx, ty, "fforeach_i")
            .unwrap();
    let v = op.pack(iv, ty.const_zero(), width, cx.repr, signed);
    cx.b.build_store(slot, v.llvm).unwrap();
    // Row-major row offset of the indices so far.
    let scaled =
        cx.b.build_int_mul(acc, i64t.const_int(count, false), "fforeach_scaled")
            .unwrap();
    let acc = cx.b.build_int_add(scaled, row_off, "fforeach_acc").unwrap();
    if vars.len() > 1 {
        fixed_outer_level(
            cx,
            &vars[1..],
            &fixed[1..],
            rest,
            src,
            acc,
            child,
            body_stmt,
            brk,
            recurse,
        )?;
    } else {
        let raw = match src {
            RowSrc::Stored(backing, base) => {
                let row = cx.b.build_int_add(base, acc, "fforeach_rowp").unwrap();
                crate::arrays::load_raw(cx, backing, row, 64)?
            }
            RowSrc::Packed(w) => {
                let sh =
                    cx.b.build_int_mul(acc, i64t.const_int(64, false), "fforeach_bits")
                        .unwrap();
                let sh =
                    cx.b.build_int_z_extend_or_bit_cast(sh, w.get_type(), "fforeach_sh")
                        .unwrap();
                let v =
                    cx.b.build_right_shift(w, sh, false, "fforeach_shr")
                        .unwrap();
                cx.b.build_int_truncate_or_bit_cast(v, i64t, "fforeach_raw")
                    .unwrap()
            }
        };
        let ch = op.lane_val(PackedValue {
            llvm: raw,
            width: 64,
            repr: cx.repr,
            signed: false,
        });
        match child {
            Some(child) => {
                foreach_coll_level(cx, rest, child, ch, body_stmt, Some(brk), recurse)?;
            }
            None => {
                // The elements are strings; `rest` is the byte dimension.
                if let Some(jvar) = rest.first().and_then(|d| d.0) {
                    let sv = crate::strings::from_storage(
                        cx,
                        PackedValue {
                            llvm: ch,
                            width: 64,
                            repr: cx.repr,
                            signed: false,
                        },
                    );
                    let len = crate::strings::str_op(
                        cx,
                        sv_sim_sim::strings::StrOp::Len,
                        sv.llvm,
                        i64t.const_zero(),
                        i64t.const_zero(),
                    );
                    lower_string_bytes_loop(cx, &jvar, len, body_stmt, Some(brk), recurse)?;
                }
            }
        }
    }
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(step_block).unwrap();
    }

    cx.b.position_at_end(step_block);
    let p =
        cx.b.build_load(i64t, pos, "fforeach_p2")
            .unwrap()
            .into_int_value();
    let next =
        cx.b.build_int_add(p, i64t.const_int(1, false), "fforeach_next")
            .unwrap();
    cx.b.build_store(pos, next).unwrap();
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(level_exit);
    Ok(())
}

/// One `foreach` dimension: its loop variable (`None` when omitted) and, for
/// a fixed-size dimension, its `(left, right)` bounds.
type ForeachDim<'d> = (Option<sv_lang::Symbol<'d>>, Option<(i32, i32)>);

/// One level of a `foreach` over a queue, dynamic or associative array `c`
/// (handle `h`): `dims[0]` indexes `c`. The dimensions after it are either
/// fixed-size ranges (`int d[][3]`) or, when their bound is dynamic, the
/// elements of `c` themselves collections (`int q[$][$]`, `int d[][]`),
/// walked by recursing into the element handle read at the current index
/// (LRM 1800-2023 §12.7.3). `brk` is the exit of the whole nest, where
/// `break` goes.
fn foreach_coll_level<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    dims: &[ForeachDim<'d>],
    c: &crate::collections::Coll<'d>,
    h: inkwell::values::IntValue<'ctx>,
    body_stmt: sv_lang::Statement<'d>,
    brk: Option<inkwell::basic_block::BasicBlock<'ctx>>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    // Fixed dimensions between this collection and a nested one
    // (`int x[][2][$]`): the nested collection is reached per combination
    // of their indices.
    let bridge = dims[1..].iter().take_while(|d| d.1.is_some()).count();
    let bridged = bridge > 0 && dims.get(1 + bridge).is_some_and(|d| d.1.is_none());
    // `string q[$]` with `foreach (q[i, j])`: `j` walks the bytes of element
    // `q[i]` (§12.7.3, §6.16), not a nested collection.
    let str_elem = !bridged && dims.len() == 2 && dims[1].1.is_none() && c.elem.is_string();
    let nested = !str_elem && (bridged || dims.get(1).is_some_and(|d| d.1.is_none()));
    let mut rest = Vec::new();
    if !nested && !str_elem {
        for (v, range) in dims.iter().skip(1) {
            let Some(v) = *v else {
                continue;
            };
            let (left, right) = range.ok_or_else(|| {
                LoweringError::unsupported_at(
                    "foreach over a dynamically-sized dimension nested inside a fixed-size \
                     dimension of a queue or dynamic array's element",
                    &cx.fn_sym,
                )
            })?;
            rest.push((v, left, right));
        }
    }
    let var = dims.first().and_then(|d| d.0);
    if str_elem && dims[1].0.is_none() {
        // `foreach (q[i, ])`: only the outer index is named, and it is
        // iterated like a plain collection.
        return foreach_coll_level(cx, &dims[..1], c, h, body_stmt, brk, recurse);
    }
    if var.is_none() && !str_elem {
        if nested {
            return Err(LoweringError::unsupported_at(
                "foreach with an omitted index in front of a nested queue or dynamic-array \
                 dimension",
                &cx.fn_sym,
            ));
        }
        // Outermost index omitted: only the named inner dimensions run.
        if rest.is_empty() {
            return Ok(());
        }
        let (_, f) = current_block_and_fn(cx)?;
        let exit = cx.ctx.append_basic_block(f, "foreach_break");
        lower_foreach_nest(cx, &rest, body_stmt, exit, recurse)?;
        if !current_block_terminated(cx) {
            cx.b.build_unconditional_branch(exit).unwrap();
        }
        cx.b.position_at_end(exit);
        return Ok(());
    }
    let assoc = c.assoc;
    let slot_info = match var {
        Some(var) => {
            declare_loop_local(cx, &var, None)?;
            Some(*cx.locals.get(&var.uid()).unwrap())
        }
        None => None,
    };
    let size = crate::collections::size_of_handle(cx, c, h);
    let i64t = cx.ctx.i64_type();
    let pos = crate::expr::entry_alloca(cx, i64t, "foreach_pos");
    cx.b.build_store(pos, i64t.const_zero()).unwrap();

    let (_, f) = current_block_and_fn(cx)?;
    let header = cx.ctx.append_basic_block(f, "cforeach_header");
    let body_block = cx.ctx.append_basic_block(f, "cforeach_body");
    let step_block = cx.ctx.append_basic_block(f, "cforeach_step");
    let exit_block = cx.ctx.append_basic_block(f, "cforeach_exit");
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(header);
    let p =
        cx.b.build_load(i64t, pos, "cforeach_p")
            .unwrap()
            .into_int_value();
    let more =
        cx.b.build_int_compare(inkwell::IntPredicate::SLT, p, size, "cforeach_more")
            .unwrap();
    cx.b.build_conditional_branch(more, body_block, exit_block)
        .unwrap();

    cx.b.position_at_end(body_block);
    let v = match slot_info {
        Some((slot, width, signed)) => {
            let v = if assoc {
                let k = crate::collections::key_at_coll(cx, c, h, p);
                crate::select::fit_width(
                    &OpBuilder {
                        ctx: cx.ctx,
                        b: &cx.b,
                    },
                    k,
                    width,
                )
            } else {
                let op = OpBuilder {
                    ctx: cx.ctx,
                    b: &cx.b,
                };
                let ty = op.i_n(width);
                let i = if width > 64 {
                    cx.b.build_int_z_extend(p, ty, "cforeach_i").unwrap()
                } else {
                    cx.b.build_int_truncate_or_bit_cast(p, ty, "cforeach_i")
                        .unwrap()
                };
                op.pack(i, ty.const_zero(), width, cx.repr, signed)
            };
            cx.b.build_store(slot, v.llvm).unwrap();
            Some(v)
        }
        None => None,
    };
    let break_to = brk.unwrap_or(exit_block);
    if bridged {
        let mut via = Vec::new();
        for (v, range) in &dims[1..=bridge] {
            let (Some(v), Some((l, r))) = (*v, *range) else {
                return Err(LoweringError::unsupported_at(
                    "foreach with an omitted index of a fixed dimension in front of a nested \
                     queue or dynamic-array dimension",
                    &cx.fn_sym,
                ));
            };
            via.push((v, l, r));
        }
        let key = assoc.then(|| v.expect("a nested dimension has a named outer index"));
        lower_foreach_nest_with(cx, &via, &mut |cx, _| {
            let (child, ch) = crate::collections::child_coll_at(cx, c, h, p, key, &via)?;
            foreach_coll_level(
                cx,
                &dims[1 + bridge..],
                &child,
                ch,
                body_stmt,
                Some(break_to),
                recurse,
            )
        })?;
    } else if nested {
        let key = assoc.then(|| v.expect("a nested dimension has a named outer index"));
        let (child, ch) = crate::collections::child_coll_at(cx, c, h, p, key, &[])?;
        foreach_coll_level(
            cx,
            &dims[1..],
            &child,
            ch,
            body_stmt,
            Some(break_to),
            recurse,
        )?;
    } else if str_elem {
        let key = assoc.then(|| v.expect("a string dimension has a named outer index"));
        let elem = crate::collections::elem_at_coll(cx, c, h, p, key)
            .into_packed("a string element", &cx.fn_sym)?;
        let sv = crate::strings::from_storage(cx, elem);
        let len = crate::strings::str_op(
            cx,
            sv_sim_sim::strings::StrOp::Len,
            sv.llvm,
            cx.ctx.i64_type().const_zero(),
            cx.ctx.i64_type().const_zero(),
        );
        let jvar = dims[1].0.expect("checked above");
        lower_string_bytes_loop(cx, &jvar, len, body_stmt, Some(break_to), recurse)?;
    } else if rest.is_empty() {
        lower_loop_body(cx, body_stmt, step_block, break_to, recurse)?;
    } else {
        lower_foreach_nest(cx, &rest, body_stmt, break_to, recurse)?;
    }
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(step_block).unwrap();
    }

    cx.b.position_at_end(step_block);
    let p =
        cx.b.build_load(i64t, pos, "cforeach_p2")
            .unwrap()
            .into_int_value();
    let next =
        cx.b.build_int_add(p, i64t.const_int(1, false), "cforeach_next")
            .unwrap();
    cx.b.build_store(pos, next).unwrap();
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(exit_block);
    Ok(())
}

/// The loops of a `foreach` ([`lower_foreach_loop`]), `loops[0]`
/// outermost, around `body`; `break` goes to `outer_exit`.
fn lower_foreach_nest<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    loops: &[(sv_lang::Symbol<'d>, i32, i32)],
    body_stmt: sv_lang::Statement<'d>,
    outer_exit: inkwell::basic_block::BasicBlock<'ctx>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    lower_foreach_nest_with(cx, loops, &mut |cx, step| match step {
        Some(step) => lower_loop_body(cx, body_stmt, step, outer_exit, recurse),
        None => recurse(cx, body_stmt),
    })
}

/// The innermost action of [`lower_foreach_nest_with`]: given the
/// innermost loop's step block (`None` when there are no loops).
type NestTail<'a, 'ctx, 'd, 'm> = dyn FnMut(
        &mut LowerCtx<'ctx, 'd, 'm>,
        Option<inkwell::basic_block::BasicBlock<'ctx>>,
    ) -> Result<(), LoweringError>
    + 'a;

/// The counted loops of [`lower_foreach_nest`] around an arbitrary
/// innermost action `tail`.
fn lower_foreach_nest_with<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    loops: &[(sv_lang::Symbol<'d>, i32, i32)],
    tail: &mut NestTail<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let Some(&(loop_var, lo, hi)) = loops.first() else {
        return tail(cx, None);
    };
    let ascending = lo <= hi;
    declare_loop_local(cx, &loop_var, None)?;
    // Overwrite the zero-initialized default with the left bound.
    let (slot, width, signed) = *cx.locals.get(&loop_var.uid()).unwrap();
    {
        let op = OpBuilder {
            ctx: cx.ctx,
            b: &cx.b,
        };
        let lo_c = op.i_n(width).const_int(lo as u64, true);
        cx.b.build_store(slot, lo_c).unwrap();
    }

    let (_, f) = current_block_and_fn(cx)?;
    let header = cx.ctx.append_basic_block(f, "foreach_header");
    let body_block = cx.ctx.append_basic_block(f, "foreach_body");
    let step_block = cx.ctx.append_basic_block(f, "foreach_step");
    let exit_block = cx.ctx.append_basic_block(f, "foreach_exit");
    cx.b.build_unconditional_branch(header).unwrap();

    cx.b.position_at_end(header);
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let ty = op.i_n(width);
    let cur =
        cx.b.build_load(ty, slot, "foreach_i")
            .unwrap()
            .into_int_value();
    let hi_c = op.i_n(width).const_int(hi as u64, true);
    let pred = if ascending {
        inkwell::IntPredicate::SLE
    } else {
        inkwell::IntPredicate::SGE
    };
    let cont =
        cx.b.build_int_compare(pred, cur, hi_c, "foreach_cont")
            .unwrap();
    cx.b.build_conditional_branch(cont, body_block, exit_block)
        .unwrap();

    cx.b.position_at_end(body_block);
    if loops.len() == 1 {
        tail(cx, Some(step_block))?;
    } else {
        lower_foreach_nest_with(cx, &loops[1..], tail)?;
    }
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(step_block).unwrap();
    }

    // The step is reached by falling off the body or by `continue`.
    cx.b.position_at_end(step_block);
    {
        let op2 = OpBuilder {
            ctx: cx.ctx,
            b: &cx.b,
        };
        let cur2 =
            cx.b.build_load(op2.i_n(width), slot, "foreach_i_reload")
                .unwrap()
                .into_int_value();
        let cur2_pv = PackedValue {
            llvm: cur2,
            width,
            repr: ValueRepr::TwoState,
            signed,
        };
        let one_pv = op2.pack(
            op2.i_n(width)
                .const_int(if ascending { 1 } else { u64::MAX }, true),
            op2.i_n(width).const_zero(),
            width,
            ValueRepr::TwoState,
            signed,
        );
        let next_pv = op2.add4(cur2_pv, one_pv);
        cx.b.build_store(slot, next_pv.llvm).unwrap();
        cx.b.build_unconditional_branch(header).unwrap();
    }

    cx.b.position_at_end(exit_block);
    Ok(())
}

/// `forever body;`: runs until a `break`, `return` or `disable`.
pub(crate) fn lower_forever_loop<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let (_, f) = current_block_and_fn(cx)?;
    let body_block = cx.ctx.append_basic_block(f, "forever_body");
    let exit_block = cx.ctx.append_basic_block(f, "forever_exit");
    cx.b.build_unconditional_branch(body_block).unwrap();
    cx.b.position_at_end(body_block);
    // VPI `cbStmt` (Table 38-6): when first encountered and every iteration.
    crate::vpi_dyn::emit_stmt_probe(cx, &s);
    let body_stmt = s
        .body()
        .ok_or_else(|| LoweringError::unsupported_at("forever missing body", &cx.fn_sym))?;
    lower_loop_body(cx, body_stmt, body_block, exit_block, recurse)?;
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(body_block).unwrap();
    }
    cx.b.position_at_end(exit_block);
    Ok(())
}

/// A `randcase`'s pick (LRM 1800-2023 §18.16): the index (an `i64`) of the
/// item a random number below the weights' sum selects, -1 for none.
///
/// Each weight has its self-determined precision and is summed as an
/// unsigned value at the widest weight's precision `W` (so a negative
/// signed weight is a large unsigned one, and the sum wraps). The number is
/// drawn with `$urandom_range(sum - 1, 0)` -- for `W` over 32 bits, one
/// `$urandom_range` per 32-bit chunk, reduced modulo the sum -- and the
/// items are taken in declaration order by unsigned `W`-bit comparison
/// against the running sum. A sum that wrapped to 0 draws from the whole
/// `W`-bit range. An unknown weight bit counts as 0. When every weight is
/// 0 no item is taken, and -- for a `randcase` at source offset `site` --
/// a warning is issued.
pub(crate) fn rand_pick<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    weights: &[sv_lang::Expression<'d>],
    site: Option<sv_lang::SourceLoc>,
) -> Result<inkwell::values::IntValue<'ctx>, LoweringError> {
    use inkwell::IntPredicate::{EQ, NE, ULT};
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let i64t = cx.ctx.i64_type();
    let i32t = cx.ctx.i32_type();
    let mut lowered = Vec::new();
    let mut width = 1u32;
    for w in weights {
        let pv = lower_expr(cx, *w)?.into_packed("a randcase weight", &cx.fn_sym)?;
        width = width.max(pv.width);
        lowered.push(op.lane_val(op.to_2state4(pv)));
    }
    let wt = op.i_n(width);
    let values: Vec<inkwell::values::IntValue<'ctx>> = lowered
        .iter()
        .map(|v| cx.b.build_int_z_extend_or_bit_cast(*v, wt, "rw").unwrap())
        .collect();
    let mut total = wt.const_zero();
    let mut any = cx.ctx.bool_type().const_zero();
    for v in &values {
        total = cx.b.build_int_add(total, *v, "rw_sum").unwrap();
        let nz =
            cx.b.build_int_compare(NE, *v, wt.const_zero(), "rw_nz")
                .unwrap();
        any = cx.b.build_or(any, nz, "rw_any").unwrap();
    }
    let range = cx
        .module
        .get_function(crate::expr::SV_RT_URANDOM_RANGE_SYMBOL)
        .unwrap_or_else(|| {
            cx.module.add_function(
                crate::expr::SV_RT_URANDOM_RANGE_SYMBOL,
                i32t.fn_type(&[i32t.into(), i32t.into()], false),
                Some(inkwell::module::Linkage::External),
            )
        });
    let urandom_range = |hi: inkwell::values::IntValue<'ctx>| {
        cx.b.build_call(range, &[hi.into(), i32t.const_zero().into()], "rw_draw")
            .unwrap()
            .try_as_basic_value()
            .unwrap_basic()
            .into_int_value()
    };
    let r = if width <= 32 {
        // `sum - 1` wraps to the whole W-bit range when the sum is 0.
        let hi =
            cx.b.build_int_sub(total, wt.const_int(1, false), "rw_hi")
                .unwrap();
        let hi =
            cx.b.build_int_z_extend_or_bit_cast(hi, i32t, "rw_hi32")
                .unwrap();
        let draw = urandom_range(hi);
        cx.b.build_int_truncate_or_bit_cast(draw, wt, "rw_r")
            .unwrap()
    } else {
        let chunks = width.div_ceil(32);
        let at = op.i_n(chunks * 32);
        let mut acc = at.const_zero();
        for _ in 0..chunks {
            let draw = urandom_range(i32t.const_all_ones());
            let draw = cx.b.build_int_z_extend(draw, at, "rw_chunk").unwrap();
            acc =
                cx.b.build_left_shift(acc, at.const_int(32, false), "rw_shl")
                    .unwrap();
            acc = cx.b.build_or(acc, draw, "rw_acc").unwrap();
        }
        let sum =
            cx.b.build_int_z_extend_or_bit_cast(total, at, "rw_sum_w")
                .unwrap();
        let wrapped =
            cx.b.build_int_compare(EQ, sum, at.const_zero(), "rw_wrapped")
                .unwrap();
        let div =
            cx.b.build_select(wrapped, at.const_int(1, false), sum, "rw_div")
                .unwrap()
                .into_int_value();
        let rem = cx.b.build_int_unsigned_rem(acc, div, "rw_rem").unwrap();
        let r =
            cx.b.build_select(wrapped, acc, rem, "rw_r")
                .unwrap()
                .into_int_value();
        cx.b.build_int_truncate_or_bit_cast(r, wt, "rw_r_w")
            .unwrap()
    };
    let mut picked = i64t.const_all_ones();
    let mut cum = wt.const_zero();
    for (k, v) in values.iter().enumerate() {
        cum = cx.b.build_int_add(cum, *v, "rw_cum").unwrap();
        let below = cx.b.build_int_compare(ULT, r, cum, "rw_below").unwrap();
        let unset =
            cx.b.build_int_compare(EQ, picked, i64t.const_all_ones(), "rw_unset")
                .unwrap();
        let take = cx.b.build_and(below, unset, "rw_take").unwrap();
        picked =
            cx.b.build_select(take, i64t.const_int(k as u64, false), picked, "rw_pick")
                .unwrap()
                .into_int_value();
    }
    let result =
        cx.b.build_select(any, picked, i64t.const_all_ones(), "rw_result")
            .unwrap()
            .into_int_value();
    if let Some(at) = site {
        let none = cx.b.build_not(any, "rw_none").unwrap();
        let (_, f) = current_block_and_fn(cx)?;
        let warn = cx.ctx.append_basic_block(f, "randcase_zero");
        let cont = cx.ctx.append_basic_block(f, "randcase_pick");
        cx.b.build_conditional_branch(none, warn, cont).unwrap();
        cx.b.position_at_end(warn);
        let sys_fns = crate::process_lower::fetch_or_declare_sys_call_fns(cx.ctx, cx.module);
        let scope = sv_sim_sim::process::cfg::shown_path(&cx.fn_sym);
        let text = crate::process_lower::build_c_string_global(
            cx.ctx,
            cx.module,
            "every weight is 0: nothing is chosen",
            "randcase_zero_msg",
        );
        let write = sys_fns.write;
        crate::process_lower::severity_frame(cx, &sys_fns, "WARNING: ", at, &scope, &mut || {
            let rtarg_ty = crate::systask::rtarg_llvm_type(cx.ctx);
            let (args_ptr, n) = crate::systask::build_rtarg_array(cx, rtarg_ty, Vec::new())?;
            cx.b.build_call(
                write,
                &[text.into(), args_ptr.into(), n.into()],
                "randcase_warn",
            )
            .unwrap();
            Ok(())
        })?;
        cx.b.build_unconditional_branch(cont).unwrap();
        cx.b.position_at_end(cont);
    }
    Ok(result)
}

/// A `randcase` whose items need no blocks of their own: lowered in place,
/// the picked item's statement run through `recurse`.
pub(crate) fn lower_randcase<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    let items = s.randcase_items();
    let weights: Vec<sv_lang::Expression<'d>> = items.iter().map(|i| i.expr()).collect();
    let idx = rand_pick(cx, &weights, Some(s.source_range().start))?;
    let (_, f) = current_block_and_fn(cx)?;
    let merge = cx.ctx.append_basic_block(f, "randcase_merge");
    let i64t = cx.ctx.i64_type();
    for (k, item) in items.iter().enumerate() {
        let arm = cx.ctx.append_basic_block(f, "randcase_arm");
        let next = cx.ctx.append_basic_block(f, "randcase_next");
        let hit =
            cx.b.build_int_compare(
                inkwell::IntPredicate::EQ,
                idx,
                i64t.const_int(k as u64, false),
                "randcase_hit",
            )
            .unwrap();
        cx.b.build_conditional_branch(hit, arm, next).unwrap();
        cx.b.position_at_end(arm);
        recurse(cx, item.stmt())?;
        if !current_block_terminated(cx) {
            cx.b.build_unconditional_branch(merge).unwrap();
        }
        cx.b.position_at_end(next);
    }
    cx.b.build_unconditional_branch(merge).unwrap();
    cx.b.position_at_end(merge);
    Ok(())
}

/// The running process ends here if it has ended the run since it resumed
/// (`$finish`/`$stop`/`$fatal` in a subroutine it called, or a null virtual
/// interface's fatal error): nothing after it may execute (LRM 1800-2023
/// §20.2/§20.10). A process kernel's segment returns `SEG_FINISHED`; any
/// other function returns at once (its caller checks in turn).
pub(crate) fn leave_if_aborted(cx: &LowerCtx) -> Result<(), LoweringError> {
    let (_, f) = current_block_and_fn(cx)?;
    let i32t = cx.ctx.i32_type();
    let rt = cx
        .module
        .get_function(crate::process_lower::SV_RT_ABORTED_SYMBOL)
        .unwrap_or_else(|| {
            cx.module.add_function(
                crate::process_lower::SV_RT_ABORTED_SYMBOL,
                i32t.fn_type(&[], false),
                None,
            )
        });
    let flag =
        cx.b.build_call(rt, &[], "aborted")
            .unwrap()
            .try_as_basic_value()
            .unwrap_basic()
            .into_int_value();
    let set =
        cx.b.build_int_compare(
            inkwell::IntPredicate::NE,
            flag,
            i32t.const_zero(),
            "is_aborted",
        )
        .unwrap();
    let leave = cx.ctx.append_basic_block(f, "aborted");
    let cont = cx.ctx.append_basic_block(f, "not_aborted");
    cx.b.build_conditional_branch(set, leave, cont).unwrap();
    cx.b.position_at_end(leave);
    leave_now(cx, f);
    cx.b.position_at_end(cont);
    Ok(())
}

/// Returns from `f` (the function being lowered) as the end of the running
/// process: see [`leave_if_aborted`].
pub(crate) fn leave_now<'ctx>(cx: &LowerCtx<'ctx, '_, '_>, f: FunctionValue<'ctx>) {
    if cx.write_through.is_some() {
        let finished = cx
            .ctx
            .i32_type()
            .const_int(crate::process_lower::SEG_FINISHED as u32 as u64, false);
        cx.b.build_return(Some(&finished)).unwrap();
        return;
    }
    match f.get_type().get_return_type() {
        Some(t) => {
            let zero = t.const_zero();
            cx.b.build_return(Some(&zero)).unwrap();
        }
        None => {
            cx.b.build_return(None).unwrap();
        }
    }
}

/// A fatal run-time error at `offset` when `cond` holds: reported like
/// `$fatal` (`FATAL: file:line: message`, then the time and scope), after
/// which the process ends ([`leave_now`]). Lowering continues on the path
/// where `cond` does not hold.
pub(crate) fn fatal_if<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    cond: inkwell::values::IntValue<'ctx>,
    at: sv_lang::SourceLoc,
    message: &str,
) -> Result<(), LoweringError> {
    let (_, f) = current_block_and_fn(cx)?;
    let bad = cx.ctx.append_basic_block(f, "fatal");
    let ok = cx.ctx.append_basic_block(f, "no_fatal");
    cx.b.build_conditional_branch(cond, bad, ok).unwrap();
    cx.b.position_at_end(bad);
    let sys_fns = crate::process_lower::fetch_or_declare_sys_call_fns(cx.ctx, cx.module);
    let scope = sv_sim_sim::process::cfg::shown_path(&cx.fn_sym);
    let text = crate::process_lower::build_c_string_global(
        cx.ctx,
        cx.module,
        &message.replace('%', "%%"),
        "fatal_msg",
    );
    let write = sys_fns.write;
    crate::process_lower::severity_frame(cx, &sys_fns, "FATAL: ", at, &scope, &mut || {
        let rtarg_ty = crate::systask::rtarg_llvm_type(cx.ctx);
        let (args_ptr, n) = crate::systask::build_rtarg_array(cx, rtarg_ty, Vec::new())?;
        cx.b.build_call(
            write,
            &[text.into(), args_ptr.into(), n.into()],
            "fatal_text",
        )
        .unwrap();
        Ok(())
    })?;
    let handle = cx
        .ctx
        .ptr_type(inkwell::AddressSpace::default())
        .const_null();
    let one = cx.ctx.i32_type().const_int(1, false);
    cx.b.build_call(sys_fns.finish, &[handle.into(), one.into()], "fatal_finish")
        .unwrap();
    leave_now(cx, f);
    cx.b.position_at_end(ok);
    Ok(())
}

/// A non-fatal run-time error at `at` when `cond` holds: reported like
/// `$error` (`ERROR: file:line: message`, then the time and scope), after
/// which execution continues on the same path (the caller supplies the
/// defined value the failed operation yields).
pub(crate) fn error_if<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    cond: inkwell::values::IntValue<'ctx>,
    at: sv_lang::SourceLoc,
    message: &str,
) -> Result<(), LoweringError> {
    report_if(cx, cond, at, message, "ERROR: ")
}

/// [`error_if`] at warning severity (`WARNING: file:line: message`): a
/// diagnostic the LRM requires that does not make the operation fail.
pub(crate) fn warning_if<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    cond: inkwell::values::IntValue<'ctx>,
    at: sv_lang::SourceLoc,
    message: &str,
) -> Result<(), LoweringError> {
    report_if(cx, cond, at, message, "WARNING: ")
}

fn report_if<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    cond: inkwell::values::IntValue<'ctx>,
    at: sv_lang::SourceLoc,
    message: &str,
    prefix: &str,
) -> Result<(), LoweringError> {
    let (_, f) = current_block_and_fn(cx)?;
    let bad = cx.ctx.append_basic_block(f, "rt_error");
    let ok = cx.ctx.append_basic_block(f, "no_rt_error");
    cx.b.build_conditional_branch(cond, bad, ok).unwrap();
    cx.b.position_at_end(bad);
    let sys_fns = crate::process_lower::fetch_or_declare_sys_call_fns(cx.ctx, cx.module);
    let scope = sv_sim_sim::process::cfg::shown_path(&cx.fn_sym);
    let text = crate::process_lower::build_c_string_global(
        cx.ctx,
        cx.module,
        &message.replace('%', "%%"),
        "rt_error_msg",
    );
    let write = sys_fns.write;
    crate::process_lower::severity_frame(cx, &sys_fns, prefix, at, &scope, &mut || {
        let rtarg_ty = crate::systask::rtarg_llvm_type(cx.ctx);
        let (args_ptr, n) = crate::systask::build_rtarg_array(cx, rtarg_ty, Vec::new())?;
        cx.b.build_call(
            write,
            &[text.into(), args_ptr.into(), n.into()],
            "rt_error_text",
        )
        .unwrap();
        Ok(())
    })?;
    cx.b.build_unconditional_branch(ok).unwrap();
    cx.b.position_at_end(ok);
    Ok(())
}

/// `if (cond) then else other` (LRM 1800-2023 §12.4) lowered in place, its
/// branches through `recurse` -- in a function, or nested in a statement a
/// process lowers as one piece (a `case` arm, a `repeat` body).
pub(crate) fn lower_if<'ctx, 'd, 'm>(
    cx: &mut LowerCtx<'ctx, 'd, 'm>,
    s: sv_lang::Statement<'d>,
    recurse: &mut Recurse<'_, 'ctx, 'd, 'm>,
) -> Result<(), LoweringError> {
    // Scope: a plain `if (expr) ...` -- the pattern-matching
    // `if (expr &&& pattern) ...` / multi-condition `&&&`-chained
    // form (`conditions().len() > 1`, or a non-`None` `pattern()`)
    // is a genuinely different construct (LRM 1800-2023 §12.4.3,
    // pattern matching) this task does not implement; a loud,
    // typed error rather than silently treating it as a plain `if`
    // on just the first condition.
    if violation_check(&s).is_some() || is_clause_if(&s) {
        let chain = if_series(s).map_err(|why| checked_if_error(why, cx))?;
        return lower_checked_arms(cx, s, &chain.thens, chain.otherwise, recurse);
    }
    let conditions = s.conditions();
    let cond_expr = conditions[0].expr();
    // A constant predicate picks its arm at elaboration: only the live arm
    // is lowered (see `constant_if_arm`).
    if let Some(taken) = sv_sim_sim::process::cfg::constant_if_arm(&s) {
        let live = if taken {
            s.then_branch()
        } else {
            s.else_branch()
        };
        if let Some(live) = live {
            recurse(cx, live)?;
        }
        return Ok(());
    }
    // Review fix (r1): route through `lower_condition`, not a bare
    // `lower_expr` + `is_definitely_true` -- when `cond_expr` is
    // itself a direct `&&`/`||` BinaryOp, that is CG-090's actual
    // "if's branch-condition site" for the operator (the
    // ARCHITECTURE.md-cited §12.4 "ambiguous -> false" collapse,
    // with its own tested short-circuit-skip-on-ambiguous
    // behavior), which must stay distinct from `&&`/`||` used as an
    // ordinary value elsewhere (see `expr.rs`'s
    // `lower_short_circuit_logical_value`, CG-021's LRM §11.4.7
    // Table 11-4 path).
    let cond_true = lower_condition(cx, cond_expr)?;

    let (_, f) = current_block_and_fn(cx)?;

    let then_block = cx.ctx.append_basic_block(f, "if_then");
    let else_block = cx.ctx.append_basic_block(f, "if_else");
    let merge_block = cx.ctx.append_basic_block(f, "if_merge");
    cx.b.build_conditional_branch(cond_true, then_block, else_block)
        .unwrap();

    // Tracks whether the merge block ends up with any real
    // predecessor at all: if BOTH branches terminate themselves
    // (e.g. `if (c) return 1; else return 2;`), the merge block is
    // unreachable dead code, which LLVM's verifier still requires
    // to end in a terminator -- `unreachable`, not left empty.
    let mut merge_reachable = false;

    cx.b.position_at_end(then_block);
    let then_stmt = s
        .then_branch()
        .ok_or_else(|| LoweringError::unsupported_at("if missing then branch", &cx.fn_sym))?;
    recurse(cx, then_stmt)?;
    // Only add the fallthrough branch to the merge block if the
    // then-branch didn't already terminate itself (e.g. via a
    // `return` -- `f.verify()` would otherwise reject a block with
    // two terminators).
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(merge_block).unwrap();
        merge_reachable = true;
    }

    cx.b.position_at_end(else_block);
    if let Some(else_stmt) = s.else_branch() {
        recurse(cx, else_stmt)?;
    }
    if !current_block_terminated(cx) {
        cx.b.build_unconditional_branch(merge_block).unwrap();
        merge_reachable = true;
    }

    cx.b.position_at_end(merge_block);
    if !merge_reachable {
        cx.b.build_unreachable().unwrap();
    }
    Ok(())
}