vj-codegen 0.1.0

//! Fixed-size unpacked arrays: element access for module memories (STATE
//! `Memory` slots, through the memory accessor) and for process/function
//! local arrays (a contiguous run of rows behind a pointer), of any number
//! of dimensions; and whole arrays as values (assignment patterns, array
//! copies, sub-arrays and slices, array-valued function calls, equality).
//!
//! An array's dimensions are flattened row-major with each dimension's
//! lowest index first (the order a memory's rows are stored in): for `T
//! a[d0][d1]...`, element `a[i0][i1]...` is row `sum_k (i_k - low_k) *
//! stride_k`. A sub-array (`m[i]` of a 2-D array) or a slice (`a[1:2]`) is
//! therefore a contiguous run of rows. An access with an unknown or
//! out-of-range index reads the element type's default (X for a 4-state
//! element, 0 for a 2-state one) and a write through it is ignored (LRM
//! 1800-2023 §7.4.6).
//!
//! Two arrays correspond element by element counting each dimension from
//! its left bound (§7.6), whatever the index directions.

use inkwell::IntPredicate;
use inkwell::values::{IntValue, PointerValue};
use sv_lang::Type;
use sv_lang::kinds::ExpressionKind;
use sv_sim_sim::symid::SymbolUid;

use crate::error::LoweringError;
use crate::expr::{LowerCtx, MemorySignalInfo, lower_expr, mem_row_index_i64};
use crate::ops::OpBuilder;
use crate::types::{TypeWidth, int_type};
use crate::value::{PackedValue, ValueRepr};

/// A fixed-size unpacked array type's dimensions (outermost first, as
/// `(left, right)`) and element type. An element itself is the shape with
/// no dimensions (one row).
pub(crate) struct ArrayShape<'d> {
    pub dims: Vec<(i64, i64)>,
    pub elem: Type<'d>,
}

impl ArrayShape<'_> {
    pub(crate) fn size(&self, k: usize) -> u64 {
        (self.dims[k].0 - self.dims[k].1).unsigned_abs() + 1
    }

    /// Total number of rows.
    pub fn rows(&self) -> u64 {
        (0..self.dims.len()).map(|k| self.size(k)).product()
    }

    /// The number of rows one step of dimension `k` spans.
    pub(crate) fn stride(&self, k: usize) -> u64 {
        (k + 1..self.dims.len()).map(|j| self.size(j)).product()
    }
}

/// The shape of `ty` if it is a fixed-size unpacked array (nested arrays
/// flattened into more dimensions).
pub(crate) fn array_shape<'d>(ty: &Type<'d>) -> Option<ArrayShape<'d>> {
    if !ty.is_unpacked_array() || ty.is_dynamically_sized_array() {
        return None;
    }
    let mut dims = Vec::new();
    let mut cur = *ty;
    while cur.is_unpacked_array() && !cur.is_dynamically_sized_array() {
        let r = cur.fixed_unpacked_array_range();
        dims.push((i64::from(r.left), i64::from(r.right)));
        cur = cur.element_type()?;
    }
    // An element may be a queue / dynamic / associative array (a handle).
    Some(ArrayShape { dims, elem: cur })
}

/// The LLVM type of one row of a local array of `elem` under `repr`.
pub(crate) fn row_type<'ctx>(
    ctx: &'ctx inkwell::context::Context,
    elem: &Type,
    repr: ValueRepr,
) -> inkwell::types::IntType<'ctx> {
    let w = elem.value_width();
    int_type(
        ctx,
        if repr == ValueRepr::FourState {
            2 * w
        } else {
            w
        },
    )
}

/// The LLVM type of a whole local array of `shape`.
pub(crate) fn local_array_type<'ctx>(
    ctx: &'ctx inkwell::context::Context,
    shape: &ArrayShape,
    repr: ValueRepr,
) -> inkwell::types::ArrayType<'ctx> {
    row_type(ctx, &shape.elem, repr).array_type(shape.rows() as u32)
}

/// The storage behind an array.
#[derive(Clone, Copy)]
pub(crate) enum Backing<'ctx> {
    /// A local array's base pointer (rows of [`row_type`]).
    Local(PointerValue<'ctx>),
    /// A module memory.
    Memory(MemorySignalInfo),
}

/// A run of stored rows: a whole array, a sub-array or slice of one, or one
/// element.
#[derive(Clone, Copy)]
pub(crate) struct ArrayRef<'ctx> {
    pub backing: Backing<'ctx>,
    /// The first row (`i64`, 0 when invalid).
    pub base: IntValue<'ctx>,
    /// Every index that selected it is known and in range (`i1`).
    pub valid: IntValue<'ctx>,
}

impl<'ctx> ArrayRef<'ctx> {
    /// All of `backing`.
    pub(crate) fn whole(cx: &LowerCtx<'ctx, '_, '_>, backing: Backing<'ctx>) -> Self {
        ArrayRef {
            backing,
            base: cx.ctx.i64_type().const_zero(),
            valid: cx.ctx.bool_type().const_all_ones(),
        }
    }

    fn always_valid(&self) -> bool {
        self.valid.get_zero_extended_constant() == Some(1)
    }
}

/// Resolves `e` -- a local array or module memory by name, possibly
/// indexed (`m[i]`, `a[i][j]`) or sliced (`a[1:2]`, `a[i+:2]`) -- to its rows
/// and its own shape (no dimensions when `e` is a single element). `None`
/// when `e` is not rooted at a local array or module memory.
pub(crate) fn resolve_array_ref<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<(ArrayRef<'ctx>, ArrayShape<'d>)>, LoweringError> {
    resolve_array_ref_in(cx, e, None)
}

/// [`resolve_array_ref`] with the root array's storage taken from `root`
/// instead of the root expression's own symbol: `vif.a[i]` (LRM 1800-2023
/// 25.9) names member `a` of every candidate instance, and `crate::vif`
/// resolves it once per candidate (`root` = that candidate's `a`).
pub(crate) fn resolve_array_ref_in<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    root: Option<&sv_lang::Symbol<'d>>,
) -> Result<Option<(ArrayRef<'ctx>, ArrayShape<'d>)>, LoweringError> {
    let mut selects = Vec::new();
    let mut cur = e;
    while matches!(
        cur.kind(),
        ExpressionKind::ElementSelect | ExpressionKind::RangeSelect
    ) {
        let Some(base) = cur.select_value() else {
            return Ok(None);
        };
        // An element of a queue, dynamic or associative array is no row of
        // this array: the walk stops at the collection.
        if !base.expr_type().is_some_and(|t| {
            t.is_unpacked_array() && !sv_sim_sim::state::storage::is_collection(&t)
        }) {
            break;
        }
        selects.push(cur);
        cur = base;
    }
    // `vif.a[i]` being resolved for one candidate instance (`crate::vif`).
    let picked = if root.is_none() {
        crate::vif::picked_symbol(&cur)
    } else {
        None
    };
    let root = root.or(picked.as_ref());
    if root.is_none() && !sv_sim_sim::symid::is_value_name(cur.kind()) {
        return Ok(None);
    }
    let Some(sym) = root.cloned().or_else(|| cur.referenced_symbol()) else {
        return Ok(None);
    };
    let backing = if let Some(info) = crate::events::static_array_backing(cx, &sym) {
        // An event array of a body with an event pool: its elements are
        // objects of the pool (`crate::events`).
        Backing::Memory(info)
    } else if let Some(info) = cx.mem_signals.get(&sym.uid()).copied() {
        Backing::Memory(info)
    } else if let Some(&(ptr, _, _)) = cx.locals.get(&sym.uid()) {
        Backing::Local(ptr)
    } else if root.is_none()
        && let Some(ptr) = constant_array(cx, &sym, cur)?
    {
        Backing::Local(ptr)
    } else {
        return Ok(None);
    };
    let Some(full) = cur.expr_type().as_ref().and_then(array_shape) else {
        return Ok(None);
    };
    selects.reverse();
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let i64t = cx.ctx.i64_type();
    let konst = |v: i64| i64t.const_int(v as u64, true);
    let mut base = i64t.const_zero();
    let mut valid = cx.ctx.bool_type().const_all_ones();
    // The current bounds of each dimension (a slice narrows its own).
    let mut dims = full.dims.clone();
    let mut k = 0;
    let index_of = |x: sv_lang::Expression<'d>| -> Result<_, LoweringError> {
        let idx = lower_expr(cx, x)?.into_packed("an array index", &cx.fn_sym)?;
        let unk = op.lane_unk(idx);
        let known =
            cx.b.build_int_compare(
                IntPredicate::EQ,
                unk,
                unk.get_type().const_zero(),
                "arr_known",
            )
            .unwrap();
        Ok((mem_row_index_i64(&op, cx, idx), known))
    };
    for sel in selects {
        if k >= dims.len() {
            return Ok(None);
        }
        let low = dims[k].0.min(dims[k].1);
        let size = (dims[k].0 - dims[k].1).unsigned_abs() + 1;
        let stride = full.stride(k);
        // The selected run's lowest index, its length, and whether it lies
        // within the dimension.
        let (lowest, count, ok) = if sel.kind() == ExpressionKind::ElementSelect {
            let Some(x) = sel.selector() else {
                return Ok(None);
            };
            let (i, known) = index_of(x)?;
            (i, 1u64, known)
        } else {
            let range_const = |x: Option<sv_lang::Expression<'d>>| {
                x.and_then(|x| x.constant_value()).and_then(|c| c.as_i64())
            };
            match sel.range_selection_kind() {
                Some(0) => {
                    let (Some(l), Some(r)) = (
                        range_const(sel.range_left()),
                        range_const(sel.range_right()),
                    ) else {
                        return Err(LoweringError::unsupported_at(
                            "an unpacked-array slice with non-constant bounds",
                            &cx.fn_sym,
                        ));
                    };
                    (
                        konst(l.min(r)),
                        (l - r).unsigned_abs() + 1,
                        cx.ctx.bool_type().const_all_ones(),
                    )
                }
                Some(kind @ (1 | 2)) => {
                    let Some(w) = range_const(sel.range_right()) else {
                        return Err(LoweringError::unsupported_at(
                            "an indexed unpacked-array slice with a non-constant width",
                            &cx.fn_sym,
                        ));
                    };
                    let Some(b) = sel.range_left() else {
                        return Ok(None);
                    };
                    let (i, known) = index_of(b)?;
                    let lowest = if kind == 1 {
                        i
                    } else {
                        cx.b.build_int_sub(i, konst(w - 1), "arr_slice_low")
                            .unwrap()
                    };
                    (lowest, w.max(1) as u64, known)
                }
                _ => return Ok(None),
            }
        };
        let norm = cx.b.build_int_sub(lowest, konst(low), "arr_norm").unwrap();
        let in_range = if count > size {
            cx.ctx.bool_type().const_zero()
        } else {
            cx.b.build_int_compare(
                IntPredicate::ULE,
                norm,
                i64t.const_int(size - count, false),
                "arr_in_range",
            )
            .unwrap()
        };
        let good = cx.b.build_and(in_range, ok, "arr_ok").unwrap();
        valid = cx.b.build_and(valid, good, "arr_valid").unwrap();
        let scaled =
            cx.b.build_int_mul(norm, i64t.const_int(stride, false), "arr_scaled")
                .unwrap();
        base = cx.b.build_int_add(base, scaled, "arr_base").unwrap();
        if sel.kind() == ExpressionKind::ElementSelect {
            k += 1;
        } else {
            // A slice keeps its dimension, now with the slice's bounds.
            let Some(r) = sel.expr_type().map(|t| t.fixed_unpacked_array_range()) else {
                return Ok(None);
            };
            dims[k] = (i64::from(r.left), i64::from(r.right));
        }
    }
    let base =
        cx.b.build_select(valid, base, i64t.const_zero(), "arr_safe_base")
            .unwrap()
            .into_int_value();
    let shape = ArrayShape {
        dims: dims[k..].to_vec(),
        elem: full.elem,
    };
    Ok(Some((
        ArrayRef {
            backing,
            base,
            valid,
        },
        shape,
    )))
}

/// A named constant fixed-size unpacked array (a `parameter`/`localparam`
/// table such as a ROM, LRM 1800-2023 6.20.1) as read-only rows laid out
/// like a local array (row = index - lowest index, per dimension), so a
/// runtime index reads it through [`Backing::Local`]. One private constant
/// global per symbol per module. `None` when `name` is not such a constant.
fn constant_array<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    sym: &sv_lang::Symbol<'d>,
    name: sv_lang::Expression<'d>,
) -> Result<Option<PointerValue<'ctx>>, LoweringError> {
    let Some(sv_lang::ConstantValue::Array(top)) = name.constant_value() else {
        return Ok(None);
    };
    let Some(shape) = name.expr_type().as_ref().and_then(array_shape) else {
        return Ok(None);
    };
    // Integral elements, and the non-integral ones that keep a stored-bits form (reals,
    // unpacked structs, nested arrays of them); a string/handle element has none.
    // A string constant has no row form: whole reads go through [`constant_string_rows`].
    if shape.elem.is_string() && !shape.elem.is_integral() {
        return Ok(None);
    }
    if !shape.elem.is_integral() && shape.elem.is_dynamically_sized_array() {
        return Err(LoweringError::unsupported_at(
            "a constant unpacked array whose elements are not integral",
            sym,
        ));
    }
    let gname = format!("__sv_sim_const_array_{}", sym.uid());
    if let Some(g) = cx.module.get_global(&gname) {
        return Ok(Some(g.as_pointer_value()));
    }
    let width = shape.elem.value_width();
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let row_ty = op.packed_type(width, cx.repr);
    let mut rows = vec![row_ty.const_zero(); shape.rows() as usize];
    // Each dimension's elements are listed from its left bound to its right.
    fn fill<'ctx>(
        cv: &sv_lang::ConstantValue,
        k: usize,
        row: u64,
        shape: &ArrayShape<'_>,
        width: u32,
        cx: &LowerCtx<'ctx, '_, '_>,
        rows: &mut [IntValue<'ctx>],
    ) -> Result<(), String> {
        if k == shape.dims.len() {
            if !shape.elem.is_integral() {
                let bits = crate::expr::const_stored_bits(&shape.elem, cv)
                    .filter(|b| b.len() as u32 == width)
                    .ok_or("an element without a stored-bits form")?;
                rows[row as usize] = crate::value::const_bits(cx.ctx, &bits, false, cx.repr).llvm;
                return Ok(());
            }
            let v = cv.as_integer().ok_or("a non-integer element")?;
            let mut bits = sv_sim_sim::state::svint_bits(v)
                .ok_or("unrecoverable element bits")?
                .into_owned();
            let pad = if v.is_signed() {
                bits.last().copied().unwrap_or(sv_lang::Bit::Zero)
            } else {
                sv_lang::Bit::Zero
            };
            bits.resize(width as usize, pad);
            rows[row as usize] =
                crate::value::const_bits(cx.ctx, &bits, v.is_signed(), cx.repr).llvm;
            return Ok(());
        }
        let sv_lang::ConstantValue::Array(elems) = cv else {
            return Err("a constant array with too few dimensions".to_string());
        };
        let (left, right) = shape.dims[k];
        let low = left.min(right);
        for (p, e) in elems.iter().enumerate() {
            let index = if left <= right {
                left + p as i64
            } else {
                left - p as i64
            };
            let r = row + (index - low) as u64 * shape.stride(k);
            fill(e, k + 1, r, shape, width, cx, rows)?;
        }
        Ok(())
    }
    fill(
        &sv_lang::ConstantValue::Array(top),
        0,
        0,
        &shape,
        width,
        cx,
        &mut rows,
    )
    .map_err(|why| {
        LoweringError::unsupported_at(&format!("constant unpacked array: {why}"), sym)
    })?;
    let g = cx
        .module
        .add_global(row_ty.array_type(rows.len() as u32), None, &gname);
    g.set_initializer(&row_ty.const_array(&rows));
    g.set_constant(true);
    g.set_linkage(inkwell::module::Linkage::Private);
    Ok(Some(g.as_pointer_value()))
}

/// An element of a named constant array of `string` or `real` (`localparam
/// string NAMES [3] = '{...}`, LRM 1800-2023 6.20.2, 7.4.6): `e` fully
/// indexes the constant, each string element a literal interned at its first
/// use. The result is a select chain over the rows keyed by the flat row
/// number; an unknown or out-of-range index reads the element type's default
/// (LRM 7.4.6): the empty string, `0.0`. `None` when `e` is not such an access.
pub(crate) fn read_constant_unpacked_element<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<crate::expr::LoweredValue<'ctx>>, LoweringError> {
    let mut selects = Vec::new();
    let mut cur = e;
    while cur.kind() == ExpressionKind::ElementSelect {
        let Some(base) = cur.select_value() else {
            return Ok(None);
        };
        selects.push(cur);
        cur = base;
    }
    if selects.is_empty() || cur.kind() != ExpressionKind::NamedValue {
        return Ok(None);
    }
    let Some(sv_lang::ConstantValue::Array(top)) = cur.constant_value() else {
        return Ok(None);
    };
    let Some(shape) = cur.expr_type().as_ref().and_then(array_shape) else {
        return Ok(None);
    };
    let is_real = shape.elem.is_floating();
    if !(shape.elem.is_string() || is_real) || selects.len() != shape.dims.len() {
        return Ok(None);
    }
    if shape.rows() > 4096 {
        return Err(LoweringError::unsupported_at(
            "a constant string/real array of more than 4096 elements indexed at run time",
            &cx.fn_sym,
        ));
    }
    selects.reverse();
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let i64t = cx.ctx.i64_type();
    let mut flat = i64t.const_zero();
    let mut valid = cx.ctx.bool_type().const_all_ones();
    for (k, sel) in selects.iter().enumerate() {
        let Some(x) = sel.selector() else {
            return Ok(None);
        };
        let idx = lower_expr(cx, x)?.into_packed("an array index", &cx.fn_sym)?;
        let unk = op.lane_unk(idx);
        let known =
            cx.b.build_int_compare(IntPredicate::EQ, unk, unk.get_type().const_zero(), "k")
                .unwrap();
        let i = mem_row_index_i64(&op, cx, idx);
        let low = shape.dims[k].0.min(shape.dims[k].1);
        let size = shape.size(k) as i64;
        let off =
            cx.b.build_int_sub(i, i64t.const_int(low as u64, true), "off")
                .unwrap();
        let in_range =
            cx.b.build_int_compare(
                IntPredicate::ULT,
                off,
                i64t.const_int(size as u64, true),
                "inr",
            )
            .unwrap();
        valid = cx.b.build_and(valid, known, "v").unwrap();
        valid = cx.b.build_and(valid, in_range, "v").unwrap();
        let term =
            cx.b.build_int_mul(off, i64t.const_int(shape.stride(k), false), "term")
                .unwrap();
        flat = cx.b.build_int_add(flat, term, "flat").unwrap();
    }
    // The constant's elements by flat row.
    fn collect(
        cv: &sv_lang::ConstantValue,
        k: usize,
        row: u64,
        shape: &ArrayShape<'_>,
        out: &mut [Option<sv_lang::ConstantValue>],
    ) -> Option<()> {
        if k == shape.dims.len() {
            if !matches!(
                cv,
                sv_lang::ConstantValue::String(_) | sv_lang::ConstantValue::Real(_)
            ) {
                return None;
            }
            out[row as usize] = Some(cv.clone());
            return Some(());
        }
        let sv_lang::ConstantValue::Array(elems) = cv else {
            return None;
        };
        let (left, right) = shape.dims[k];
        let low = left.min(right);
        for (p, el) in elems.iter().enumerate() {
            let index = if left <= right {
                left + p as i64
            } else {
                left - p as i64
            };
            collect(
                el,
                k + 1,
                row + (index - low) as u64 * shape.stride(k),
                shape,
                out,
            )?;
        }
        Some(())
    }
    let mut texts = vec![None; shape.rows() as usize];
    collect(
        &sv_lang::ConstantValue::Array(top),
        0,
        0,
        &shape,
        &mut texts,
    )
    .ok_or_else(|| {
        LoweringError::unsupported_at(
            "a constant string/real array with an element of another kind",
            &cx.fn_sym,
        )
    })?;
    let f64t = cx.ctx.f64_type();
    let mut acc: inkwell::values::BasicValueEnum<'ctx> = if is_real {
        f64t.const_zero().into()
    } else {
        i64t.const_zero().into()
    };
    let default = acc;
    for (r, cv) in texts.iter().enumerate() {
        let Some(cv) = cv else { continue };
        let lit: inkwell::values::BasicValueEnum<'ctx> = match cv {
            sv_lang::ConstantValue::Real(f) => f64t.const_float(*f).into(),
            sv_lang::ConstantValue::String(text) => {
                crate::strings::literal(cx, text.as_bytes()).llvm.into()
            }
            _ => unreachable!("checked in collect"),
        };
        let hit =
            cx.b.build_int_compare(
                IntPredicate::EQ,
                flat,
                i64t.const_int(r as u64, false),
                "hit",
            )
            .unwrap();
        acc = cx.b.build_select(hit, lit, acc, "cconst").unwrap();
    }
    let acc =
        cx.b.build_select(valid, acc, default, "cconst_valid")
            .unwrap();
    Ok(Some(if is_real {
        crate::expr::LoweredValue::Real(crate::value::RealValue {
            llvm: acc.into_float_value(),
        })
    } else {
        crate::expr::LoweredValue::Str(crate::strings::StrValue {
            llvm: acc.into_int_value(),
        })
    }))
}

/// One addressed element: its storage, row (`i64`, 0 when invalid) and
/// whether the access is in range with known indices.
pub(crate) struct ElemRef<'ctx> {
    pub backing: Backing<'ctx>,
    pub row: IntValue<'ctx>,
    pub valid: IntValue<'ctx>,
    pub width: u32,
    pub signed: bool,
    /// The element type's default value (read for an invalid access).
    pub default: IntValue<'ctx>,
}

/// Resolves `e` -- `a[i]`, `a[i][j]`, ... fully indexing a local array or a
/// module memory -- to its element, or `None` when `e` is not such an
/// access (a bit-select of a packed value, a sub-array, ...).
pub(crate) fn resolve_element<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<ElemRef<'ctx>>, LoweringError> {
    resolve_element_in(cx, e, None)
}

/// [`resolve_element`] over `root`'s storage ([`resolve_array_ref_in`]).
pub(crate) fn resolve_element_in<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    root: Option<&sv_lang::Symbol<'d>>,
) -> Result<Option<ElemRef<'ctx>>, LoweringError> {
    if e.kind() != ExpressionKind::ElementSelect
        || e.expr_type().is_some_and(|t| {
            t.is_unpacked_array() && !sv_sim_sim::state::storage::is_collection(&t)
        })
    {
        return Ok(None);
    }
    // A named event element is no value: only a trigger, a wait,
    // `.triggered`, an assignment or a comparison use it
    // (`resolve_event_element`, `crate::events`).
    if e.expr_type().is_some_and(|t| t.is_event()) {
        return Err(LoweringError::unsupported_at(
            "a named event array element used as a value",
            &cx.fn_sym,
        ));
    }
    let Some((r, shape)) = resolve_array_ref_in(cx, e, root)? else {
        return Ok(None);
    };
    if !shape.dims.is_empty() {
        return Ok(None);
    }
    Ok(Some(ElemRef {
        backing: r.backing,
        row: r.base,
        valid: r.valid,
        width: shape.elem.value_width(),
        signed: shape.elem.is_signed(),
        default: default_value(cx, &shape.elem),
    }))
}

/// The storage and shape of `e` if it is a whole or partial array held in a
/// local array or module memory (an assignment target or a stored array
/// value); `None` for an element or anything else.
pub(crate) fn array_target<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<(ArrayRef<'ctx>, ArrayShape<'d>)>, LoweringError> {
    let e = peel_array_conversions(e);
    if !e.expr_type().is_some_and(|t| t.is_unpacked_array()) {
        return Ok(None);
    }
    Ok(resolve_array_ref(cx, e)?.filter(|(_, shape)| !shape.dims.is_empty()))
}

/// A READ-ONLY local copy of the fixed-size unpacked array `e` that is a
/// member (a class property or an unpacked-struct field, LRM 1800-2023 7.4,
/// 8.5) and so has no row storage of its own: its flat stored value (row `r`
/// in bits `[r*w +: w]`, the form [`pack_array_value`] produces) is cut into
/// the rows of a fresh local array. `None` when `e` is not such a member or
/// its elements hold a class handle or collection (the copy would need
/// reference counting). The copy is a snapshot: a write to it never reaches
/// the member, so only a caller that reads may use it.
pub(crate) fn spill_member_array<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<(ArrayRef<'ctx>, ArrayShape<'d>)>, LoweringError> {
    let e = peel_array_conversions(e);
    // `obj.arr` / `s.arr`, or a bare `arr` naming a non-static class property
    // inside a method (implicit `this.arr`, LRM 8.11).
    let is_member = e.kind() == ExpressionKind::MemberAccess
        || (e.kind() == ExpressionKind::NamedValue
            && e.referenced_symbol()
                .is_some_and(|s| s.kind() == sv_lang::kinds::SymbolKind::ClassProperty));
    if !is_member {
        return Ok(None);
    }
    let Some(shape) = e.expr_type().as_ref().and_then(array_shape) else {
        return Ok(None);
    };
    if shape.dims.is_empty()
        || shape.rows() > MAX_UNROLLED_ROWS
        || !crate::collections::handle_offsets(&shape.elem).is_empty()
    {
        return Ok(None);
    }
    let flat = crate::expr::lower_expr(cx, e)?.into_packed("a member array value", &cx.fn_sym)?;
    let ptr = rows_from_flat(cx, flat, &shape, "arr_member")?;
    Ok(Some((ArrayRef::whole(cx, Backing::Local(ptr)), shape)))
}

/// A fresh local array of `shape` whose rows are cut out of the flat value `flat` (row `r` in
/// bits `[r*w +: w]`, the form [`pack_array_value`] produces); the base pointer of the rows
/// ([`Backing::Local`]). A snapshot: writing the rows never reaches `flat`'s source.
pub(crate) fn rows_from_flat<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    flat: PackedValue<'ctx>,
    shape: &ArrayShape<'_>,
    name: &str,
) -> Result<PointerValue<'ctx>, LoweringError> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let ptr = crate::expr::entry_alloca(cx, local_array_type(cx.ctx, shape, cx.repr), name);
    let tmp = ArrayRef::whole(cx, Backing::Local(ptr));
    let w = shape.elem.value_width();
    let i64t = cx.ctx.i64_type();
    for r in 0..shape.rows() {
        let row = PackedValue {
            signed: shape.elem.is_signed(),
            ..op.part_select4(
                flat,
                op.i_n(flat.width).const_int(r * u64::from(w), false),
                cx.ctx.bool_type().const_zero(),
                w,
            )
        };
        store_row(cx, &tmp, i64t.const_int(r, false), row)?;
    }
    Ok(ptr)
}

/// The temporary an array-valued user call `e` returned into, with its shape
/// (a call has no storage of its own to resolve; `%p` of one reads this).
/// `None` when `e` is not such a call.
pub(crate) fn array_call_value<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<(ArrayRef<'ctx>, ArrayShape<'d>)>, LoweringError> {
    let e = peel_array_conversions(e);
    if crate::func::user_callee(&e).is_none() {
        return Ok(None);
    }
    let Some(shape) = e.expr_type().as_ref().and_then(array_shape) else {
        return Ok(None);
    };
    let ptr = crate::func::lower_array_call(cx, e)?;
    Ok(Some((ArrayRef::whole(cx, Backing::Local(ptr)), shape)))
}

/// The pointer to row `row` of a local array.
pub(crate) fn local_row_ptr<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    base: PointerValue<'ctx>,
    row: IntValue<'ctx>,
    width: u32,
) -> PointerValue<'ctx> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    // SAFETY-free GEP: `row` is in range or 0 (`resolve_array_ref`).
    unsafe {
        cx.b.build_in_bounds_gep(op.packed_type(width, cx.repr), base, &[row], "arr_elem")
    }
    .unwrap()
}

/// The element's default value (X for a 4-state element, else 0).
pub(crate) fn default_value<'ctx>(cx: &LowerCtx<'ctx, '_, '_>, value_ty: &Type) -> IntValue<'ctx> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let width = value_ty.value_width();
    // An unpacked struct/union mixes 4-state (X) and 2-state (0) fields.
    if sv_sim_sim::state::storage::is_unpacked_aggregate(value_ty)
        && let Some(bits) = sv_sim_sim::state::storage::default_bits(value_ty)
    {
        return crate::value::const_bits(cx.ctx, &bits, false, cx.repr).llvm;
    }
    let ty = op.packed_type(width, cx.repr);
    if value_ty.is_four_state() && cx.repr == ValueRepr::FourState {
        // `(unk << width) | val` with every unk bit set: all X.
        let ones = ty.const_all_ones();
        cx.b.build_left_shift(ones, ty.const_int(u64::from(width), false), "arr_default_x")
            .unwrap()
    } else {
        ty.const_zero()
    }
}

/// A whole-array operation (copy, compare, concatenation, a stream) reaching a
/// memory whose elements are wider than the LLVM-integer ceiling: only an
/// element access is supported (`crate::huge::mem_elem_root`).
fn huge_rows_refused(cx: &LowerCtx<'_, '_, '_>, width: u32) -> Result<(), LoweringError> {
    if width > crate::value::WIDE_THRESHOLD {
        return Err(LoweringError::unsupported_at(
            "a whole-array operation on an unpacked array whose elements are wider than the \
             wide-value threshold (LRM 1800-2023 6.9.1); only reading or writing one element \
             (`m[i]`, `m[i][b]`) is supported",
            &cx.fn_sym,
        ));
    }
    Ok(())
}

/// Loads row `row` (in range) of `backing`'s `width`-bit elements.
pub(crate) fn load_raw<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    backing: Backing<'ctx>,
    row: IntValue<'ctx>,
    width: u32,
) -> Result<IntValue<'ctx>, LoweringError> {
    Ok(match backing {
        Backing::Memory(info) => {
            huge_rows_refused(cx, width)?;
            let acc = cx.mem_accessor.ok_or_else(|| {
                LoweringError::unsupported_at("memory read with no accessor context", &cx.fn_sym)
            })?;
            crate::accessor::build_get_mem_word(cx, acc.fns, acc.handle, info.slot, row, width)
        }
        Backing::Local(base) => {
            let op = OpBuilder {
                ctx: cx.ctx,
                b: &cx.b,
            };
            let p = local_row_ptr(cx, base, row, width);
            cx.b.build_load(op.packed_type(width, cx.repr), p, "arr_ld")
                .unwrap()
                .into_int_value()
        }
    })
}

/// Stores `value` to row `row` (in range) of `backing`'s `width`-bit
/// elements.
pub(crate) fn store_raw<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    backing: Backing<'ctx>,
    row: IntValue<'ctx>,
    value: IntValue<'ctx>,
    width: u32,
) -> Result<(), LoweringError> {
    match backing {
        Backing::Memory(info) => {
            huge_rows_refused(cx, width)?;
            let acc = cx.mem_accessor.ok_or_else(|| {
                LoweringError::unsupported_at("memory write with no accessor context", &cx.fn_sym)
            })?;
            crate::accessor::build_set_mem_word(
                cx, acc.fns, acc.handle, info.slot, row, value, width,
            );
        }
        Backing::Local(base) => {
            let p = local_row_ptr(cx, base, row, width);
            cx.b.build_store(p, value).unwrap();
        }
    }
    Ok(())
}

/// Row `k` of `r` (elements of type `elem`): its value, or the element
/// default when `r` is invalid.
pub(crate) fn load_row<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    r: &ArrayRef<'ctx>,
    k: IntValue<'ctx>,
    elem: &Type,
) -> Result<PackedValue<'ctx>, LoweringError> {
    let width = elem.value_width();
    let row = cx.b.build_int_add(r.base, k, "arr_row").unwrap();
    let fetched = load_raw(cx, r.backing, row, width)?;
    let llvm = if r.always_valid() {
        fetched
    } else {
        let dflt = default_value(cx, elem);
        cx.b.build_select(r.valid, fetched, dflt, "arr_read")
            .unwrap()
            .into_int_value()
    };
    Ok(PackedValue {
        llvm,
        width,
        repr: cx.repr,
        signed: elem.is_signed(),
    })
}

/// Every element of the stored array `e` (a local array or module memory,
/// whole or a sub-array), in bit-stream order -- each dimension's left
/// bound first (LRM 1800-2023 §11.4.14.1) -- with the element type; `None`
/// when `e` is not a stored array.
pub(crate) fn stored_elements_in_stream_order<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<(Vec<PackedValue<'ctx>>, Type<'d>)>, LoweringError> {
    let Some((r, shape)) = array_target(cx, peel_array_conversions(e))? else {
        return Ok(None);
    };
    let i64t = cx.ctx.i64_type();
    let mut out = Vec::with_capacity(shape.rows() as usize);
    for flat in 0..shape.rows() {
        // Declared order: the i-th element of a dimension is its left bound
        // stepping toward its right; rows hold the lowest index first.
        let mut rest = flat;
        let mut row = 0u64;
        for k in (0..shape.dims.len()).rev() {
            let size = shape.size(k);
            let i = rest % size;
            rest /= size;
            let (left, right) = shape.dims[k];
            let pos = if left <= right { i } else { size - 1 - i };
            row += pos * shape.stride(k);
        }
        out.push(load_row(cx, &r, i64t.const_int(row, false), &shape.elem)?);
    }
    Ok(Some((out, shape.elem)))
}

/// Runs `f` only when `valid` holds.
pub(crate) fn guarded<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    valid: IntValue<'ctx>,
    f: impl FnOnce() -> Result<(), LoweringError>,
) -> Result<(), LoweringError> {
    if valid.get_zero_extended_constant() == Some(1) {
        return f();
    }
    let func =
        cx.b.get_insert_block()
            .and_then(|b| b.get_parent())
            .expect("inside a function");
    let then_bb = cx.ctx.append_basic_block(func, "arr_valid");
    let done_bb = cx.ctx.append_basic_block(func, "arr_valid_done");
    cx.b.build_conditional_branch(valid, then_bb, done_bb)
        .unwrap();
    cx.b.position_at_end(then_bb);
    f()?;
    cx.b.build_unconditional_branch(done_bb).unwrap();
    cx.b.position_at_end(done_bb);
    Ok(())
}

/// Stores `value` (already the element width) to row `k` of `r`; nothing
/// when `r` is invalid.
pub(crate) fn store_row<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    r: &ArrayRef<'ctx>,
    k: IntValue<'ctx>,
    value: PackedValue<'ctx>,
) -> Result<(), LoweringError> {
    guarded(cx, r.valid, || {
        let row = cx.b.build_int_add(r.base, k, "arr_row").unwrap();
        store_raw(cx, r.backing, row, value.llvm, value.width)
    })
}

/// Reads element `r` (its default when the access is invalid).
pub(crate) fn read_element<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    r: &ElemRef<'ctx>,
) -> Result<PackedValue<'ctx>, LoweringError> {
    let fetched = load_raw(cx, r.backing, r.row, r.width)?;
    let dflt = r.default;
    let v =
        cx.b.build_select(r.valid, fetched, dflt, "arr_read")
            .unwrap()
            .into_int_value();
    Ok(PackedValue {
        llvm: v,
        width: r.width,
        repr: cx.repr,
        signed: r.signed,
    })
}

/// Writes `value` (already `r.width` bits) to element `r`; an invalid
/// access writes nothing.
pub(crate) fn write_element<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    r: &ElemRef<'ctx>,
    value: PackedValue<'ctx>,
) -> Result<(), LoweringError> {
    guarded(cx, r.valid, || {
        store_raw(cx, r.backing, r.row, value.llvm, r.width)
    })
}

/// One addressed element `ea[i]` of an unpacked array of named events
/// (LRM 1800-2023 15.5, 6.17): a row of the module memory `ea`, holding the
/// event's 128-bit cell (`sv_sim_sim::state::EVENT_CELL_WIDTH`).
#[derive(Clone, Copy)]
pub(crate) struct EventElem<'ctx> {
    pub info: MemorySignalInfo,
    /// The element's row (`i64`, 0 when invalid).
    pub row: IntValue<'ctx>,
    /// Every index is known and in range (`i1`). An invalid element is no
    /// event: a trigger does nothing, a wait never wakes (7.4.6).
    pub valid: IntValue<'ctx>,
}

/// Resolves `e` -- `ea[i]`, `ea[i][j]`, ... fully indexing a module's array
/// of named events -- to its element, or `None` when `e` is not one.
pub(crate) fn resolve_event_element<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<Option<EventElem<'ctx>>, LoweringError> {
    if let Some(h) = crate::events::resolve_handle(cx, e)? {
        return Ok(Some(h));
    }
    if e.kind() != ExpressionKind::ElementSelect || !e.expr_type().is_some_and(|t| t.is_event()) {
        return Ok(None);
    }
    let Some((r, shape)) = resolve_array_ref(cx, e)? else {
        return Ok(None);
    };
    if !shape.dims.is_empty() {
        return Ok(None);
    }
    match r.backing {
        Backing::Memory(info) => Ok(Some(EventElem {
            info,
            row: r.base,
            valid: r.valid,
        })),
        Backing::Local(_) => Err(LoweringError::unsupported_at(
            "an element of an event array that is not a module-level signal",
            &cx.fn_sym,
        )),
    }
}

/// The packed value (`i(2N)`, an all-zero unknown plane) of event element
/// `el`'s cell; the row 0 cell when the element is invalid (read it only
/// under `el.valid`).
pub(crate) fn load_event_elem<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    el: &EventElem<'ctx>,
) -> Result<IntValue<'ctx>, LoweringError> {
    load_raw(
        cx,
        Backing::Memory(el.info),
        el.row,
        sv_sim_sim::state::EVENT_CELL_WIDTH,
    )
}

/// Stores the packed cell value `value` to event element `el` (through the
/// memory accessor, which wakes the memory's waiters).
pub(crate) fn store_event_elem<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    el: &EventElem<'ctx>,
    value: IntValue<'ctx>,
) -> Result<(), LoweringError> {
    store_raw(
        cx,
        Backing::Memory(el.info),
        el.row,
        value,
        sv_sim_sim::state::EVENT_CELL_WIDTH,
    )
}

/// Runs `f` only when `valid` holds.
pub(crate) fn when_valid<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    valid: IntValue<'ctx>,
    f: impl FnOnce() -> Result<(), LoweringError>,
) -> Result<(), LoweringError> {
    guarded(cx, valid, f)
}

/// Emits `for (k = 0; k < rows; k++) body(k)` with `k` an `i64`, threading
/// one accumulator `acc` (of `acc`'s type) through the iterations; returns
/// its final value.
pub(crate) fn row_loop<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    rows: u64,
    acc: IntValue<'ctx>,
    mut body: impl FnMut(IntValue<'ctx>, IntValue<'ctx>) -> Result<IntValue<'ctx>, LoweringError>,
) -> Result<IntValue<'ctx>, LoweringError> {
    let i64t = cx.ctx.i64_type();
    let pre = cx.b.get_insert_block().expect("positioned");
    let func = pre.get_parent().expect("inside a function");
    let head = cx.ctx.append_basic_block(func, "arr_loop");
    let body_bb = cx.ctx.append_basic_block(func, "arr_loop_row");
    let done = cx.ctx.append_basic_block(func, "arr_loop_done");
    cx.b.build_unconditional_branch(head).unwrap();
    cx.b.position_at_end(head);
    let k = cx.b.build_phi(i64t, "k").unwrap();
    k.add_incoming(&[(&i64t.const_zero(), pre)]);
    let a = cx.b.build_phi(acc.get_type(), "acc").unwrap();
    a.add_incoming(&[(&acc, pre)]);
    let kv = k.as_basic_value().into_int_value();
    let av = a.as_basic_value().into_int_value();
    let more =
        cx.b.build_int_compare(IntPredicate::ULT, kv, i64t.const_int(rows, false), "more")
            .unwrap();
    cx.b.build_conditional_branch(more, body_bb, done).unwrap();
    cx.b.position_at_end(body_bb);
    let next_acc = body(kv, av)?;
    let next =
        cx.b.build_int_add(kv, i64t.const_int(1, false), "k_next")
            .unwrap();
    let latch = cx.b.get_insert_block().expect("positioned");
    k.add_incoming(&[(&next, latch)]);
    a.add_incoming(&[(&next_acc, latch)]);
    cx.b.build_unconditional_branch(head).unwrap();
    cx.b.position_at_end(done);
    Ok(av)
}

/// Sets every row of the local array at `base` (`shape`) to its element
/// default (X or 0) -- a local array's initial value.
pub(crate) fn init_local_array<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    base: PointerValue<'ctx>,
    shape: &ArrayShape,
) {
    let width = shape.elem.value_width();
    let dflt = default_value(cx, &shape.elem);
    let unit = cx.ctx.bool_type().const_zero();
    row_loop(cx, shape.rows(), unit, |k, acc| {
        let p = local_row_ptr(cx, base, k, width);
        cx.b.build_store(p, dflt).unwrap();
        Ok(acc)
    })
    .expect("a local store cannot fail");
}

/// Gives the local array at `ptr` (shape `shape`) its starting value: its
/// declared initializer, or every element's default.
pub(crate) fn init_array_var<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    ptr: PointerValue<'ctx>,
    shape: &ArrayShape<'d>,
    init: Option<sv_lang::Expression<'d>>,
) -> Result<(), LoweringError> {
    match init {
        // The storage is being given its first value: it holds no
        // references to give back.
        Some(init) => {
            assign_array_into_fresh(cx, ArrayRef::whole(cx, Backing::Local(ptr)), shape, init)
        }
        None => {
            init_local_array(cx, ptr, shape);
            Ok(())
        }
    }
}

/// A whole fixed-size unpacked array used as a value.
enum ArraySource<'ctx, 'd> {
    /// Stored rows (a local array or module memory, or part of one, or the
    /// temporary an array-valued call returned into).
    Stored(ArrayRef<'ctx>),
    /// An assignment pattern: one expression per element, in row order.
    Rows(Vec<Row<'d>>),
    /// Already-computed element values, in row order (the elements of a
    /// streamed or bit-stream-cast array value).
    Values(Vec<PackedValue<'ctx>>),
}

/// One element of an assignment pattern's row list.
#[derive(Clone, Copy)]
enum Row<'d> {
    /// An element given by an expression.
    Expr(sv_lang::Expression<'d>),
    /// One character of a string literal assigned to a `byte` array (LRM
    /// 1800-2023 5.9: left-justified, null-padded on the right, truncated
    /// on the right).
    Byte(u8),
}

/// The value of one pattern row converted to the element type `elem`.
fn row_value<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    r: Row<'d>,
    elem: &Type,
    own: bool,
) -> Result<PackedValue<'ctx>, LoweringError> {
    match r {
        Row::Expr(x) => element_value(cx, x, elem, own),
        Row::Byte(c) => {
            let op = OpBuilder {
                ctx: cx.ctx,
                b: &cx.b,
            };
            let bits: Vec<sv_lang::Bit> = (0..8)
                .map(|i| {
                    if c >> i & 1 == 1 {
                        sv_lang::Bit::One
                    } else {
                        sv_lang::Bit::Zero
                    }
                })
                .collect();
            let v = crate::value::const_bits(cx.ctx, &bits, false, cx.repr);
            Ok(crate::select::fit_width(&op, v, elem.value_width()))
        }
    }
}

/// The text of a string literal used as an unpacked-array value, if `e` is
/// one (a `Conversion` of a `StringLiteral` to a one-dimensional `byte`
/// array, LRM 1800-2023 5.9).
fn string_literal_bytes(e: sv_lang::Expression<'_>) -> Option<Vec<u8>> {
    let mut x = e;
    while x.kind() == ExpressionKind::Conversion {
        x = x.conversion_operand()?;
    }
    if x.kind() != ExpressionKind::StringLiteral {
        return None;
    }
    x.string_literal_value().map(String::into_bytes)
}

/// `e` with implicit conversions between array types peeled off.
fn peel_array_conversions(mut e: sv_lang::Expression<'_>) -> sv_lang::Expression<'_> {
    while e.kind() == ExpressionKind::Conversion
        && let Some(inner) = e.conversion_operand()
        && inner.expr_type().is_some_and(|t| t.is_unpacked_array())
    {
        e = inner;
    }
    e
}

/// Resolves an unpacked-array-typed expression -- a stored array (whole,
/// sub-array or slice), an array-valued user function call, or an
/// assignment pattern (`'{...}`, `'{default: v}`, `'{n{...}}`, nested per
/// dimension) -- to its shape and rows.
fn array_operand<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<(ArrayShape<'d>, ArraySource<'ctx, 'd>), LoweringError> {
    let e = peel_array_conversions(e);
    let shape = e
        .expr_type()
        .as_ref()
        .and_then(array_shape)
        .ok_or_else(|| {
            LoweringError::unsupported_at(
                "an unpacked-array value whose type is not a fixed-size array",
                &cx.fn_sym,
            )
        })?;
    if let Some((r, _)) = array_target(cx, e)? {
        return Ok((shape, ArraySource::Stored(r)));
    }
    // A sub-array or slice of a virtual interface's array member
    // (`vif.a[2:3]`, `vif.b[1]`): each candidate's rows, the selected one's.
    if let Some(values) = crate::vif::array_values(cx, e) {
        return Ok((shape, ArraySource::Values(values?)));
    }
    // A fixed array that is a class property / struct member has no row
    // storage (its value lives flat in the object): snapshot it into a local
    // row array, as the array methods do.
    if let Some((r, _)) = spill_member_array(cx, e)? {
        return Ok((shape, ArraySource::Stored(r)));
    }
    // `{>>{...}}` / `T'(v)` into an array: the value's stored bits, lowest
    // index least significant at every level, so row `r` (row-major,
    // lowest index first) is simply the `r`-th element-width slice.
    if e.kind() == ExpressionKind::Conversion && matches!(e.conversion_kind(), Some(2 | 4)) {
        let stored = if e.conversion_kind() == Some(2) {
            crate::stream::lower_stream_conversion(cx, e)?
        } else {
            crate::stream::lower_bitstream_cast(cx, e)?
        }
        .into_packed("a streamed array value", &cx.fn_sym)?;
        let op = OpBuilder {
            ctx: cx.ctx,
            b: &cx.b,
        };
        let ew = shape.elem.value_width();
        let values = (0..shape.rows() as u32)
            .map(|r| PackedValue {
                signed: shape.elem.is_signed(),
                ..op.part_select4(
                    stored,
                    op.i_n(stored.width).const_int(u64::from(r * ew), false),
                    cx.ctx.bool_type().const_zero(),
                    ew,
                )
            })
            .collect();
        return Ok((shape, ArraySource::Values(values)));
    }
    // An element of a queue, dynamic or associative array that is itself a
    // fixed-size array (`int w[$][2]`; `a = w[i]`): its stored row, cut into
    // one slice per element exactly like a streamed array value.
    if e.kind() == ExpressionKind::ElementSelect
        && e.select_value()
            .is_some_and(|b| crate::collections::coll_of_expr(&b).is_some())
    {
        let stored =
            crate::expr::lower_expr(cx, e)?.into_packed("a collection element", &cx.fn_sym)?;
        let op = OpBuilder {
            ctx: cx.ctx,
            b: &cx.b,
        };
        let ew = shape.elem.value_width();
        let values = (0..shape.rows() as u32)
            .map(|r| PackedValue {
                signed: shape.elem.is_signed(),
                ..op.part_select4(
                    stored,
                    op.i_n(stored.width).const_int(u64::from(r * ew), false),
                    cx.ctx.bool_type().const_zero(),
                    ew,
                )
            })
            .collect();
        return Ok((shape, ArraySource::Values(values)));
    }
    // A slice of a dynamic array or queue (`d[a:b]`, `d[a+:w]`; LRM 1800-2023
    // 7.4.3, 10.10): a fixed-size array whose element `k` is `d[a + k]`.
    if e.kind() == ExpressionKind::RangeSelect
        && let Some(base) = e.select_value()
        && let Some(c) = crate::collections::coll_of_expr(&base)
        && !c.assoc
        && shape.dims.len() == 1
        && !shape.elem.is_floating()
        && crate::collections::handle_offsets(&shape.elem).is_empty()
    {
        let h = crate::collections::handle_of(cx, base)?;
        let (lo, _) = crate::collections::slice_bounds(cx, &c, h, e)?;
        let i64t = cx.ctx.i64_type();
        let width = shape.elem.value_width();
        let op = OpBuilder {
            ctx: cx.ctx,
            b: &cx.b,
        };
        let mut values = Vec::new();
        for k in 0..shape.rows() {
            let pos =
                cx.b.build_int_add(lo, i64t.const_int(k, false), "slice_pos")
                    .unwrap();
            let v = crate::collections::read_at_coll(cx, &c, h, pos)
                .into_packed("a dynamic array slice element", &cx.fn_sym)?;
            values.push(crate::select::fit_width(&op, v, width));
        }
        if shape.dims[0].0 > shape.dims[0].1 {
            values.reverse();
        }
        return Ok((shape, ArraySource::Values(values)));
    }
    if let Some(values) = crate::array_methods::map_fixed_values(cx, e) {
        return Ok((shape, ArraySource::Values(values?)));
    }
    if crate::func::user_callee(&e).is_some() {
        let ptr = crate::func::lower_array_call(cx, e)?;
        return Ok((
            shape,
            ArraySource::Stored(ArrayRef::whole(cx, Backing::Local(ptr))),
        ));
    }
    if e.kind() == ExpressionKind::ConditionalOp {
        let values = conditional_array_values(cx, e, &shape)?;
        return Ok((shape, ArraySource::Values(values)));
    }
    if e.kind() == ExpressionKind::Concatenation
        && e.operands()
            .iter()
            .any(|o| o.expr_type().is_some_and(|t| t.is_unpacked_array()))
    {
        let values = array_concat_values(cx, e, &shape)?;
        return Ok((shape, ArraySource::Values(values)));
    }
    // A named constant array of `string` (LRM 1800-2023 6.20.2) read whole: each
    // element's literal, in row order.
    if e.kind() == ExpressionKind::NamedValue
        && shape.elem.is_string()
        && let Some(sv_lang::ConstantValue::Array(top)) = e.constant_value()
    {
        let values = constant_string_rows(cx, &sv_lang::ConstantValue::Array(top), &shape)?;
        return Ok((shape, ArraySource::Values(values)));
    }
    let mut rows = Vec::new();
    pattern_rows(cx, e, &shape, 0, &mut rows)?;
    Ok((shape, ArraySource::Rows(rows)))
}

/// The elements of the constant string array `cv` as stored values in row
/// order (lowest index first at every dimension); each is its literal's id.
fn constant_string_rows<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    cv: &sv_lang::ConstantValue,
    shape: &ArrayShape<'_>,
) -> Result<Vec<PackedValue<'ctx>>, LoweringError> {
    fn walk<'ctx>(
        cx: &LowerCtx<'ctx, '_, '_>,
        cv: &sv_lang::ConstantValue,
        k: usize,
        shape: &ArrayShape<'_>,
        out: &mut Vec<PackedValue<'ctx>>,
    ) -> Result<(), LoweringError> {
        let bad = || {
            LoweringError::unsupported_at(
                "a constant string array with an element of another kind",
                &cx.fn_sym,
            )
        };
        if k == shape.dims.len() {
            let sv_lang::ConstantValue::String(text) = cv else {
                return Err(bad());
            };
            let lit = crate::strings::literal(cx, text.as_bytes());
            let v = crate::real::to_storage(
                cx,
                crate::expr::LoweredValue::Str(lit),
                shape.elem.value_width(),
                "a string array element",
            )?;
            out.push(v);
            return Ok(());
        }
        let sv_lang::ConstantValue::Array(elems) = cv else {
            return Err(bad());
        };
        // The elements run from the left bound; rows are lowest index first.
        let mut parts: Vec<Vec<PackedValue<'ctx>>> = Vec::new();
        for el in elems.iter() {
            let mut one = Vec::new();
            walk(cx, el, k + 1, shape, &mut one)?;
            parts.push(one);
        }
        if shape.dims[k].0 > shape.dims[k].1 {
            parts.reverse();
        }
        out.extend(parts.into_iter().flatten());
        Ok(())
    }
    let mut out = Vec::new();
    walk(cx, cv, 0, shape, &mut out)?;
    Ok(out)
}

/// The elements of the aggregate conditional `c ? a : b` over fixed-size
/// unpacked arrays (LRM 1800-2023 11.2.2 lists `?:` among the operators
/// valid on aggregates), in `shape`'s row order. A known condition selects
/// one arm's element; an ambiguous (X/Z) one combines the two arms
/// element by element exactly like the scalar form (11.4.11, bits that
/// differ become X). Both arms are read, so an arm that is an array-valued
/// call, or whose elements own handles, is refused by name rather than
/// evaluated on the path the condition did not choose.
fn conditional_array_values<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    shape: &ArrayShape<'d>,
) -> Result<Vec<PackedValue<'ctx>>, LoweringError> {
    let refuse = |what: &str| {
        LoweringError::unsupported_at(&format!("an unpacked-array `?:` whose {what}"), &cx.fn_sym)
    };
    let conds = e.conditions();
    if conds.len() != 1 || conds[0].pattern().is_some() {
        return Err(refuse("condition is a `matches` pattern"));
    }
    if !crate::collections::handle_offsets(&shape.elem).is_empty() {
        return Err(refuse("elements are class handles or collections"));
    }
    let arm = |x: Option<sv_lang::Expression<'d>>| {
        x.ok_or_else(|| refuse("arm is missing"))
            .and_then(|x| {
                let inner = peel_array_conversions(x);
                if crate::func::user_callee(&inner).is_some() {
                    return Err(refuse("arm is an array-valued call"));
                }
                if inner.kind() == ExpressionKind::ConditionalOp {
                    return Ok((conditional_array_values(cx, inner, shape)?, true));
                }
                Ok((Vec::new(), false))
            })
            .map(|(v, done)| (v, done, x))
    };
    let (tv, t_done, t) = arm(e.true_value())?;
    let (fv, f_done, f) = arm(e.false_value())?;
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let i64t = cx.ctx.i64_type();
    let width = shape.elem.value_width();
    let fit = |v: PackedValue<'ctx>| crate::select::fit_width(&op, v, width);
    let rows_of = |done: bool,
                   vals: Vec<PackedValue<'ctx>>,
                   x: Option<sv_lang::Expression<'d>>|
     -> Result<Vec<PackedValue<'ctx>>, LoweringError> {
        if done {
            return Ok(vals);
        }
        let (s, src) = array_operand(cx, x.expect("checked by arm"))?;
        check_compatible(cx, shape, &s)?;
        let all: Vec<PackedValue<'ctx>> = match src {
            ArraySource::Rows(exprs) => exprs
                .iter()
                .map(|&r| row_value(cx, r, &shape.elem, false))
                .collect::<Result<_, _>>()?,
            ArraySource::Values(vs) => vs.into_iter().map(fit).collect(),
            ArraySource::Stored(sr) => (0..s.rows())
                .map(|r| load_row(cx, &sr, i64t.const_int(r, false), &s.elem).map(fit))
                .collect::<Result<_, _>>()?,
        };
        Ok((0..shape.rows())
            .map(|r| all[corresponding_row(shape, &s, r) as usize])
            .collect())
    };
    let then_rows = rows_of(t_done, tv, t)?;
    let else_rows = rows_of(f_done, fv, f)?;
    let cond_pv = crate::huge::truth_packed(
        cx,
        crate::expr::lower_expr_h(cx, conds[0].expr())?,
        crate::huge::TruthMode::Ternary,
        "ternary condition must be an integral (non-real) value",
    )?;
    let cond_true = crate::ops::ternary_is_definitely_true(cx.ctx, &cx.b, cond_pv);
    let cond_false = crate::ops::is_definitely_false(cx.ctx, &cx.b, cond_pv);
    Ok(then_rows
        .into_iter()
        .zip(else_rows)
        .map(|(a, b)| {
            let merged = op.ternary_merge4(a, b);
            let inner = op.select(cond_false, b, merged);
            op.select(cond_true, a, inner)
        })
        .collect())
}

/// Whether a value of type `ty` holds a queue, dynamic or associative array
/// (not merely class handles): an unpacked array concatenation cannot tell
/// such an element from a spliced operand by type alone, so those targets
/// stay refused by name.
fn holds_collection(ty: &Type<'_>) -> bool {
    crate::collections::handle_offsets(ty)
        .iter()
        .any(|s| s.kind == crate::collections::HandleKind::Coll)
}

/// Whether an unpacked array concatenation item of type `ty` is ONE element
/// of a target whose elements hold queues, dynamic or associative arrays
/// (LRM 1800-2023 10.10: an item whose type is assignment compatible with the
/// element type is a single element; only otherwise is it an array whose
/// elements are spliced in). `int q[$]` is one element of `int t[2][$]`;
/// `int qq[$][$]` or `int u[2][$]` splice.
fn is_single_collection_element(elem: &Type<'_>, ty: Option<&Type<'_>>) -> bool {
    holds_collection(elem) && ty.is_some_and(|t| elem.is_assignment_compatible(t))
}

/// The elements of the fixed-size one-dimensional array operand `o`, each
/// converted to `elem`, in left-bound-first order (the order an unpacked array
/// concatenation consumes them, LRM 1800-2023 10.10). Multi-dimensional
/// operands are refused by name.
pub(crate) fn array_operand_elements<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    o: sv_lang::Expression<'d>,
    elem: &sv_lang::Type<'d>,
) -> Result<Vec<PackedValue<'ctx>>, LoweringError> {
    array_operand_elements_with(cx, o, elem, false)
}

/// [`array_operand_elements`]; with `own`, every element carries its own
/// reference to the class handles it holds (retained when read from a stored
/// array that stays as it is), for a store that gives them to a target.
fn array_operand_elements_with<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    o: sv_lang::Expression<'d>,
    elem: &sv_lang::Type<'d>,
    own: bool,
) -> Result<Vec<PackedValue<'ctx>>, LoweringError> {
    let (os, mut vs) = array_operand_rows(cx, o, elem, own)?;
    if os.dims.len() != 1 {
        return Err(LoweringError::unsupported_at(
            "an unpacked array concatenation with a multi-dimensional operand",
            &cx.fn_sym,
        ));
    }
    // An operand's elements run from its left bound, so a descending one
    // reverses.
    if os.dims[0].0 > os.dims[0].1 {
        vs.reverse();
    }
    Ok(vs)
}

/// The rows of the fixed-size unpacked array operand `o` (any rank), each
/// converted to `elem`, in row order (row-major, lowest index first) with
/// its shape. With `own`, each value carries its own reference to the class
/// handles it holds (see [`array_operand_elements_with`]); a fresh value (an
/// array-valued call's result, a pattern's `new`) passes through as it is.
fn array_operand_rows<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    o: sv_lang::Expression<'d>,
    elem: &sv_lang::Type<'d>,
    own: bool,
) -> Result<(ArrayShape<'d>, Vec<PackedValue<'ctx>>), LoweringError> {
    let (os, src) = array_operand(cx, o)?;
    let i64t = cx.ctx.i64_type();
    let retains = own
        && crate::func::user_callee(&peel_array_conversions(o)).is_none()
        && !crate::collections::handle_offsets(&os.elem).is_empty();
    let vs = match src {
        ArraySource::Rows(rows) => rows
            .iter()
            .map(|&x| row_value(cx, x, elem, own))
            .collect::<Result<Vec<_>, _>>()?,
        ArraySource::Values(vs) => vs,
        ArraySource::Stored(sr) => (0..os.rows())
            .map(|k| {
                let v = load_row(cx, &sr, i64t.const_int(k, false), &os.elem)?;
                Ok(if retains {
                    crate::collections::retain_stored(cx, &os.elem, v)
                } else {
                    v
                })
            })
            .collect::<Result<Vec<_>, LoweringError>>()?,
    };
    Ok((os, vs))
}

/// The element values of an unpacked array concatenation `{a, b, ...}` some
/// of whose operands are themselves fixed-size one-dimensional arrays (LRM
/// 1800-2023 10.10: each array operand contributes its elements in index
/// order, each other operand contributes one element), in row order of the
/// result `shape`. Collection operands, multi-dimensional shapes and
/// handle elements (which would need per-element retention) are refused.
fn array_concat_values<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    shape: &ArrayShape<'d>,
) -> Result<Vec<PackedValue<'ctx>>, LoweringError> {
    let refuse = |what: &str| {
        Err(LoweringError::unsupported_at(
            &format!("an unpacked array concatenation with {what}"),
            &cx.fn_sym,
        ))
    };
    // Class-handle elements: every value is handed on owning its references.
    let own = !crate::collections::handle_offsets(&shape.elem).is_empty();
    if shape.dims.len() != 1 {
        return array_concat_values_nd(cx, e, shape, own);
    }
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let width = shape.elem.value_width();
    let mut values = Vec::new();
    for o in e.operands() {
        let ty = o.expr_type();
        if is_single_collection_element(&shape.elem, ty.as_ref()) {
            // A whole queue / dynamic / associative array that is one element
            // of the target: the row owns its own copy.
            values.push(element_value(cx, o, &shape.elem, true)?);
            continue;
        }
        if ty
            .as_ref()
            .is_some_and(sv_sim_sim::state::storage::is_collection)
        {
            return refuse("a queue, dynamic or associative array operand");
        }
        if ty.as_ref().is_some_and(|t| t.is_unpacked_array()) {
            let vs = array_operand_elements_with(cx, o, &shape.elem, own)?;
            values.extend(
                vs.into_iter()
                    .map(|v| crate::select::fit_width(&op, v, width)),
            );
        } else {
            values.push(element_value(cx, o, &shape.elem, own)?);
        }
    }
    if values.len() as u64 != shape.rows() {
        return refuse("an element count that differs from the target array's size");
    }
    // `values` is in left-bound-first order; rows are lowest index first.
    if shape.dims[0].0 > shape.dims[0].1 {
        values.reverse();
    }
    Ok(values)
}

/// [`array_concat_values`] for a multi-dimensional result (LRM 1800-2023
/// 10.10): the result's element type is its sub-array type (`int[2]` for
/// `int[4][2]`), so an operand one rank lower contributes one such element
/// and an operand of the result's own rank contributes each of its
/// outermost elements, left bound first. Returned in row order.
fn array_concat_values_nd<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    shape: &ArrayShape<'d>,
    own: bool,
) -> Result<Vec<PackedValue<'ctx>>, LoweringError> {
    let refuse = |what: &str| {
        Err(LoweringError::unsupported_at(
            &format!("an unpacked array concatenation with {what}"),
            &cx.fn_sym,
        ))
    };
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let width = shape.elem.value_width();
    let inner = shape.stride(0) as usize;
    // Outermost elements, each `inner` rows, left bound first.
    let mut chunks: Vec<Vec<PackedValue<'ctx>>> = Vec::new();
    for o in e.operands() {
        let ty = o.expr_type();
        if ty
            .as_ref()
            .is_some_and(sv_sim_sim::state::storage::is_collection)
        {
            return refuse("a queue, dynamic or associative array operand");
        }
        if !ty.as_ref().is_some_and(|t| t.is_unpacked_array()) {
            return refuse("a non-array operand in a multi-dimensional result");
        }
        let (os, vs) = array_operand_rows(cx, o, &shape.elem, own)?;
        let vs: Vec<_> = vs
            .into_iter()
            .map(|v| crate::select::fit_width(&op, v, width))
            .collect();
        if os.dims.len() + 1 == shape.dims.len() && vs.len() == inner {
            chunks.push(vs);
        } else if os.dims.len() == shape.dims.len() && vs.len() % inner == 0 {
            let mut cs: Vec<Vec<PackedValue<'ctx>>> = vs.chunks(inner).map(<[_]>::to_vec).collect();
            if os.dims[0].0 > os.dims[0].1 {
                cs.reverse();
            }
            chunks.extend(cs);
        } else {
            return refuse("an operand whose rank or element shape differs from the result's");
        }
    }
    if chunks.len() as u64 != shape.size(0) {
        return refuse("an element count that differs from the target array's size");
    }
    if shape.dims[0].0 > shape.dims[0].1 {
        chunks.reverse();
    }
    Ok(chunks.into_iter().flatten().collect())
}

/// The fixed-size unpacked array `e` as ONE packed value -- the form an
/// element of a queue, dynamic or associative array takes (`w.push_back(a)`
/// for `int w[$][2]`): row `r` (row-major, lowest index first) in bits
/// `[r*w +: w]`. The value carries its own reference to every collection or
/// class handle its elements hold, which the collection then owns.
pub(crate) fn pack_array_value<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
) -> Result<PackedValue<'ctx>, LoweringError> {
    let (shape, src) = array_operand(cx, e)?;
    if shape.rows() > MAX_UNROLLED_ROWS {
        return Err(LoweringError::unsupported_at(
            "a very large fixed-size array stored as a collection element",
            &cx.fn_sym,
        ));
    }
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let width = shape.elem.value_width();
    let share = crate::func::user_callee(&peel_array_conversions(e)).is_none();
    let retains = share && !crate::collections::handle_offsets(&shape.elem).is_empty();
    let mut values = match src {
        ArraySource::Rows(exprs) => exprs
            .iter()
            .map(|&x| row_value(cx, x, &shape.elem, true))
            .collect::<Result<Vec<_>, _>>()?,
        ArraySource::Values(vs) => vs
            .into_iter()
            .map(|v| crate::select::fit_width(&op, v, width))
            .collect(),
        ArraySource::Stored(sr) => (0..shape.rows())
            .map(|r| {
                let v = load_row(cx, &sr, cx.ctx.i64_type().const_int(r, false), &shape.elem)?;
                Ok(if retains {
                    crate::collections::retain_stored(cx, &shape.elem, v)
                } else {
                    v
                })
            })
            .collect::<Result<Vec<_>, LoweringError>>()?,
    };
    values.reverse();
    Ok(op.concat4(&values))
}

/// Stores the flat value `v` -- row `r` (row-major, lowest index first) in
/// bits `[r*w +: w]`, the form [`pack_array_value`] produces -- into every row
/// of the fixed-size unpacked array `target` (a local or module array whose
/// elements hold no class handle or collection). The inverse of
/// [`pack_array_value`]; `std::randomize` brings an array's solved elements
/// back this way (LRM 18.12).
pub(crate) fn store_array_from_packed<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    target: sv_lang::Expression<'d>,
    v: PackedValue<'ctx>,
) -> Result<(), LoweringError> {
    let Some((dst, shape)) = array_target(cx, target)? else {
        return Err(LoweringError::unsupported_at(
            "an array that is not stored as rows",
            &cx.fn_sym,
        ));
    };
    if !crate::collections::handle_offsets(&shape.elem).is_empty()
        || shape.rows() > MAX_UNROLLED_ROWS
    {
        return Err(LoweringError::unsupported_at(
            "storing a flat value into an array of handles or a very large array",
            &cx.fn_sym,
        ));
    }
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let w = shape.elem.value_width();
    let i64t = cx.ctx.i64_type();
    for r in 0..shape.rows() {
        let row = PackedValue {
            signed: false,
            ..op.part_select4(
                v,
                op.i_n(v.width).const_int(r * u64::from(w), false),
                cx.ctx.bool_type().const_zero(),
                w,
            )
        };
        store_row(cx, &dst, i64t.const_int(r, false), row)?;
    }
    Ok(())
}

/// The nonblocking form of [`store_array_from_packed`]: row `r` of the module
/// memory `target` takes bits `[r*w +: w]` of `v` through `write` as a
/// memory-row target, so the caller stages it (LRM 1800-2023 11.4.14.3).
pub(crate) fn write_array_rows_from_packed<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    target: sv_lang::Expression<'d>,
    v: PackedValue<'ctx>,
    write: &mut dyn FnMut(
        crate::select::Lvalue<'ctx>,
        PackedValue<'ctx>,
    ) -> Result<(), LoweringError>,
) -> Result<(), LoweringError> {
    let Some((dst, shape)) = array_target(cx, target)? else {
        return Err(LoweringError::unsupported_at(
            "an array that is not stored as rows",
            &cx.fn_sym,
        ));
    };
    let Backing::Memory(info) = dst.backing else {
        return Err(LoweringError::unsupported_at(
            "a nonblocking assignment to a local array",
            &cx.fn_sym,
        ));
    };
    if !crate::collections::handle_offsets(&shape.elem).is_empty()
        || shape.rows() > MAX_UNROLLED_ROWS
    {
        return Err(LoweringError::unsupported_at(
            "storing a flat value into an array of handles or a very large array",
            &cx.fn_sym,
        ));
    }
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let w = shape.elem.value_width();
    let i64t = cx.ctx.i64_type();
    for r in 0..shape.rows() {
        let row = PackedValue {
            signed: false,
            ..op.part_select4(
                v,
                op.i_n(v.width).const_int(r * u64::from(w), false),
                cx.ctx.bool_type().const_zero(),
                w,
            )
        };
        guarded(cx, dst.valid, || {
            let idx =
                cx.b.build_int_add(dst.base, i64t.const_int(r, false), "arr_row")
                    .unwrap();
            write(
                crate::select::mem_row_lvalue(cx, info, idx, row.width, row.signed),
                row,
            )
        })?;
    }
    Ok(())
}

/// Appends the element expressions of pattern `e` (for dimension `k` of
/// `shape` and every dimension inside it) in row order.
fn pattern_rows<'d>(
    cx: &LowerCtx<'_, 'd, '_>,
    e: sv_lang::Expression<'d>,
    shape: &ArrayShape<'d>,
    k: usize,
    out: &mut Vec<Row<'d>>,
) -> Result<(), LoweringError> {
    let e = peel_array_conversions(e);
    // A string literal fills the innermost dimension of a `byte` array.
    if k + 1 == shape.dims.len()
        && shape.elem.value_width() == 8
        && let Some(bytes) = string_literal_bytes(e)
    {
        let n = shape.size(k) as usize;
        let mut chars: Vec<Row<'d>> = (0..n)
            .map(|i| Row::Byte(bytes.get(i).copied().unwrap_or(0)))
            .collect();
        if shape.dims[k].0 > shape.dims[k].1 {
            chars.reverse();
        }
        out.extend(chars);
        return Ok(());
    }
    let mut elems = match e.kind() {
        ExpressionKind::SimpleAssignmentPattern | ExpressionKind::StructuredAssignmentPattern => {
            e.pattern_elements()
        }
        // An unpacked array concatenation of elements (`{a, b, c}`, LRM
        // 1800-2023 §10.10) fills the array like a pattern.
        ExpressionKind::Concatenation
            if e.operands().iter().all(|o| {
                !o.expr_type().is_some_and(|t| {
                    t.is_unpacked_array() || sv_sim_sim::state::storage::is_collection(&t)
                })
            }) =>
        {
            e.operands()
        }
        ExpressionKind::ReplicatedAssignmentPattern => {
            let n = e
                .replicated_pattern_count()
                .and_then(|c| c.constant_value())
                .and_then(|c| c.as_i64())
                .ok_or_else(|| {
                    LoweringError::unsupported_at(
                        "a replicated assignment pattern with a non-constant count",
                        &cx.fn_sym,
                    )
                })?;
            let one = e.pattern_elements();
            (0..n).flat_map(|_| one.iter().copied()).collect()
        }
        _ => {
            return Err(LoweringError::unsupported_at(
                &format!(
                    "an unpacked-array value of expression kind {:?} (only a stored \
                     array, an array-valued function call or an assignment pattern)",
                    e.kind()
                ),
                &cx.fn_sym,
            ));
        }
    };
    if elems.len() as u64 != shape.size(k) {
        return Err(LoweringError::unsupported_at(
            "an assignment pattern whose element count does not match its array dimension",
            &cx.fn_sym,
        ));
    }
    // Pattern elements run from the left bound; rows are lowest index first.
    if shape.dims[k].0 > shape.dims[k].1 {
        elems.reverse();
    }
    for el in elems {
        if k + 1 == shape.dims.len() {
            out.push(Row::Expr(el));
        } else {
            pattern_rows(cx, el, shape, k + 1, out)?;
        }
    }
    Ok(())
}

/// The row of an array of shape `from` holding the element at the same
/// position (counted from each dimension's left bound) as row `row` of an
/// array of shape `to`. Both shapes have the same dimension sizes.
fn corresponding_row(to: &ArrayShape, from: &ArrayShape, row: u64) -> u64 {
    let mut rest = row;
    let mut out = 0;
    let mut stride = 1;
    for k in (0..to.dims.len()).rev() {
        let size = to.size(k);
        let off = rest % size;
        rest /= size;
        let flip = (to.dims[k].0 > to.dims[k].1) != (from.dims[k].0 > from.dims[k].1);
        out += if flip { size - 1 - off } else { off } * stride;
        stride *= size;
    }
    out
}

/// Whether row `r` of one shape always corresponds to row `r` of the other.
fn same_row_order(a: &ArrayShape, b: &ArrayShape) -> bool {
    (0..a.dims.len()).all(|k| (a.dims[k].0 > a.dims[k].1) == (b.dims[k].0 > b.dims[k].1))
}

/// Largest array handled element by element (a pattern, or a copy between
/// arrays of opposite index directions); others copy in a loop.
pub(crate) const MAX_UNROLLED_ROWS: u64 = 1 << 16;

fn check_compatible(
    cx: &LowerCtx<'_, '_, '_>,
    a: &ArrayShape,
    b: &ArrayShape,
) -> Result<(), LoweringError> {
    let same = a.dims.len() == b.dims.len() && (0..a.dims.len()).all(|k| a.size(k) == b.size(k));
    if !same {
        return Err(LoweringError::unsupported_at(
            "unpacked arrays of different shapes in one assignment or comparison",
            &cx.fn_sym,
        ));
    }
    if a.rows() > MAX_UNROLLED_ROWS && !same_row_order(a, b) {
        return Err(LoweringError::unsupported_at(
            "a copy or comparison of a very large array between opposite index directions",
            &cx.fn_sym,
        ));
    }
    Ok(())
}

/// One element expression's value as `elem`'s stored bits. With `own`, the
/// value carries its own reference to every object it names (a value read
/// from some other holder is retained, a fresh one is not: a class handle
/// element stored into an array, LRM 1800-2023 8.29), for a store that
/// gives it to the element; without it, the value is only compared.
fn element_value<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    elem: &Type,
    own: bool,
) -> Result<PackedValue<'ctx>, LoweringError> {
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let width = elem.value_width();
    let lowered = if own {
        crate::collections::owned(cx, e)?
    } else {
        lower_expr(cx, e)?
    };
    let v = crate::real::to_storage(cx, lowered, width, "an array element wider than 128 bits")?;
    Ok(crate::select::fit_width(&op, v, width))
}

/// Whether two stored arrays may share rows.
fn may_overlap(a: &ArrayRef<'_>, b: &ArrayRef<'_>) -> bool {
    match (a.backing, b.backing) {
        (Backing::Local(x), Backing::Local(y)) => x == y,
        (Backing::Memory(x), Backing::Memory(y)) => x.slot == y.slot,
        _ => false,
    }
}

/// Copies `src`'s rows into a fresh local temporary, so a copy between two
/// overlapping runs of one array reads every element before writing any.
/// With `retain`, the temporary takes its own reference to every object and
/// collection its rows name (the copy then overwrites the rows it came
/// from, and must not free what a later row still has to read).
fn snapshot<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    src: &ArrayRef<'ctx>,
    shape: &ArrayShape,
    retain: bool,
) -> Result<ArrayRef<'ctx>, LoweringError> {
    let ptr = crate::expr::entry_alloca(cx, local_array_type(cx.ctx, shape, cx.repr), "arr_tmp");
    let tmp = ArrayRef::whole(cx, Backing::Local(ptr));
    let unit = cx.ctx.bool_type().const_zero();
    row_loop(cx, shape.rows(), unit, |k, acc| {
        let mut v = load_row(cx, src, k, &shape.elem)?;
        if retain {
            v = crate::collections::retain_stored(cx, &shape.elem, v);
        }
        store_row(cx, &tmp, k, v)?;
        Ok(acc)
    })?;
    Ok(tmp)
}

/// Evaluates every element of the array value `src` (shape `src_shape`)
/// and hands each, as `dst_shape`'s element, to `write` with the row of
/// `dst_shape` it belongs at. Every element is read before any is written:
/// a stored source that may share rows with `dst` is first snapshotted.
///
/// Ownership (classes Task 10, LRM 1800-2023 8.29): an element holding class
/// handles or collections owns one reference to each. Every value handed to
/// `write` carries its own such references, to be given to the element it is
/// stored into -- retained here when it is read from a stored array that
/// stays as it is (`share`), a fresh value's (a pattern's `new`, the array an
/// array-valued call returned) passed through as they are (`!share`).
fn emit_rows<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    dst: Option<&ArrayRef<'ctx>>,
    dst_shape: &ArrayShape<'d>,
    src_shape: &ArrayShape<'d>,
    src: ArraySource<'ctx, 'd>,
    share: bool,
    mut write: impl FnMut(IntValue<'ctx>, PackedValue<'ctx>) -> Result<(), LoweringError>,
) -> Result<(), LoweringError> {
    check_compatible(cx, dst_shape, src_shape)?;
    let retains = share && !crate::collections::handle_offsets(&src_shape.elem).is_empty();
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let i64t = cx.ctx.i64_type();
    let width = dst_shape.elem.value_width();
    let fit = |v: PackedValue<'ctx>| crate::select::fit_width(&op, v, width);
    let values = match src {
        ArraySource::Rows(exprs) => exprs
            .iter()
            .map(|&x| row_value(cx, x, &dst_shape.elem, true))
            .collect::<Result<Vec<_>, _>>()?,
        ArraySource::Values(vs) => vs.into_iter().map(fit).collect(),
        ArraySource::Stored(sr) => {
            // Rows that may be overwritten before they are read are
            // snapshotted holding their own references; the rest are
            // retained as they are read.
            let (sr, retain_read) = match dst {
                Some(d) if may_overlap(d, &sr) => (snapshot(cx, &sr, src_shape, retains)?, false),
                _ => (sr, retains),
            };
            let read = |k: IntValue<'ctx>| -> Result<PackedValue<'ctx>, LoweringError> {
                let v = load_row(cx, &sr, k, &src_shape.elem)?;
                Ok(fit(if retain_read {
                    crate::collections::retain_stored(cx, &src_shape.elem, v)
                } else {
                    v
                }))
            };
            if same_row_order(dst_shape, src_shape) {
                let unit = cx.ctx.bool_type().const_zero();
                row_loop(cx, dst_shape.rows(), unit, |k, acc| {
                    write(k, read(k)?)?;
                    Ok(acc)
                })?;
                return Ok(());
            }
            (0..src_shape.rows())
                .map(|r| read(i64t.const_int(r, false)))
                .collect::<Result<Vec<_>, LoweringError>>()?
        }
    };
    for r in 0..dst_shape.rows() {
        let v = values[corresponding_row(dst_shape, src_shape, r) as usize];
        write(i64t.const_int(r, false), v)?;
    }
    Ok(())
}

/// `dst = rhs` for an unpacked array `dst` (all or part of a local array or
/// module memory, of shape `dst_shape`): every element of the right-hand
/// side is evaluated before any is written, so `a = '{a[1], a[0]}` swaps.
///
/// Ownership (classes Task 10, LRM 1800-2023 8.29): an element holding class
/// handles or collections owns one reference to each. The elements copied
/// in each take their own reference (`b = a; a[0] = null;` leaves `b[0]`
/// alive), and the references the overwritten elements held are given back.
pub(crate) fn assign_array<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    dst: ArrayRef<'ctx>,
    dst_shape: &ArrayShape<'d>,
    rhs: sv_lang::Expression<'d>,
) -> Result<(), LoweringError> {
    assign_array_with(cx, dst, dst_shape, rhs, true)
}

/// [`assign_array`] into storage that holds nothing yet (a declaration's
/// initializer, a call's argument temporary: uninitialised memory whose
/// handles must not be released).
pub(crate) fn assign_array_into_fresh<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    dst: ArrayRef<'ctx>,
    dst_shape: &ArrayShape<'d>,
    rhs: sv_lang::Expression<'d>,
) -> Result<(), LoweringError> {
    assign_array_with(cx, dst, dst_shape, rhs, false)
}

/// Gives back the references row `k` of `r` (elements of type `elem`) holds.
fn release_row<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    r: &ArrayRef<'ctx>,
    k: IntValue<'ctx>,
    elem: &Type,
) -> Result<(), LoweringError> {
    let old = load_row(cx, r, k, elem)?;
    crate::collections::release_handles(cx, elem, crate::expr::LoweredValue::Packed(old));
    Ok(())
}

fn assign_array_with<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    dst: ArrayRef<'ctx>,
    dst_shape: &ArrayShape<'d>,
    rhs: sv_lang::Expression<'d>,
    holds_old: bool,
) -> Result<(), LoweringError> {
    let has_handles = !crate::collections::handle_offsets(&dst_shape.elem).is_empty();
    let releases = holds_old && has_handles;
    // A queue or dynamic array's elements, from the left bound (a missing
    // element leaves its element type's default).
    let inner = peel_array_conversions(rhs);
    if crate::collections::coll_of_expr(&inner).is_some() {
        if dst_shape.dims.len() != 1 {
            return Err(LoweringError::unsupported_at(
                "a queue or dynamic array assigned to a multi-dimensional array",
                &cx.fn_sym,
            ));
        }
        let rows = dst_shape.rows();
        let descending = dst_shape.dims[0].0 > dst_shape.dims[0].1;
        let h = crate::collections::handle_of(cx, inner)?;
        let i64t = cx.ctx.i64_type();
        let op = OpBuilder {
            ctx: cx.ctx,
            b: &cx.b,
        };
        let unit = cx.ctx.bool_type().const_zero();
        // LRM 1800-2023 7.6: copying a queue or dynamic array into a
        // fixed-size array of a different size is a run-time error and no
        // operation is performed (the target keeps its value).
        let coll = crate::collections::coll_of_expr(&inner).expect("checked above");
        let size = crate::collections::size_of_handle(cx, &coll, h);
        let size =
            cx.b.build_int_z_extend_or_bit_cast(size, i64t, "src_size")
                .unwrap();
        let bad =
            cx.b.build_int_compare(
                IntPredicate::NE,
                size,
                i64t.const_int(rows, false),
                "size_bad",
            )
            .unwrap();
        let func =
            cx.b.get_insert_block()
                .and_then(|b| b.get_parent())
                .expect("in a function");
        let err_bb = cx.ctx.append_basic_block(func, "arr_size_err");
        let copy_bb = cx.ctx.append_basic_block(func, "arr_size_ok");
        let done_bb = cx.ctx.append_basic_block(func, "arr_size_done");
        cx.b.build_conditional_branch(bad, err_bb, copy_bb).unwrap();
        cx.b.position_at_end(err_bb);
        crate::stmt_shared::error_if(
            cx,
            cx.ctx.bool_type().const_int(1, false),
            rhs.source_range().start,
            &format!(
                "a queue or dynamic array assigned to a fixed-size array of {rows} elements has a different size"
            ),
        )?;
        cx.b.build_unconditional_branch(done_bb).unwrap();
        cx.b.position_at_end(copy_bb);
        row_loop(cx, rows, unit, |k, acc| {
            let pos = if descending {
                cx.b.build_int_sub(i64t.const_int(rows - 1, false), k, "arr_from_pos")
                    .unwrap()
            } else {
                k
            };
            let v = crate::collections::read_at(cx, inner, h, pos)?;
            let width = dst_shape.elem.value_width();
            let v = crate::real::to_storage(cx, v, width, "a >128-bit array element")?;
            // The collection keeps its reference; the element takes its own.
            let v = crate::collections::retain_stored(
                cx,
                &dst_shape.elem,
                crate::select::fit_width(&op, v, width),
            );
            if releases {
                release_row(cx, &dst, k, &dst_shape.elem)?;
            }
            store_row(cx, &dst, k, v)?;
            Ok(acc)
        })?;
        cx.b.build_unconditional_branch(done_bb).unwrap();
        cx.b.position_at_end(done_bb);
        return Ok(());
    }
    if inner.kind() == ExpressionKind::Concatenation
        && inner.operands().iter().any(|o| {
            let ty = o.expr_type();
            ty.as_ref()
                .is_some_and(sv_sim_sim::state::storage::is_collection)
                && !is_single_collection_element(&dst_shape.elem, ty.as_ref())
        })
    {
        return assign_concat_with_collections(cx, &dst, dst_shape, inner, releases);
    }
    let (src_shape, src) = array_operand(cx, rhs)?;
    // An array-valued call's result is a temporary whose references pass to
    // `dst`; any other stored array stays as it is.
    let share = crate::func::user_callee(&inner).is_none();
    emit_rows(cx, Some(&dst), dst_shape, &src_shape, src, share, |k, v| {
        if releases {
            release_row(cx, &dst, k, &dst_shape.elem)?;
        }
        store_row(cx, &dst, k, v)
    })
}

/// `dst = {a, q, d, x, ...}` for a fixed-size one-dimensional `dst` whose
/// concatenation has whole queue, dynamic or associative array operands (LRM 1800-2023
/// 10.10: each array operand contributes its elements in index order, each
/// other operand one element). The element count is only known at run time:
/// when it differs from the target's size the assignment is a run-time error
/// and no operation is performed (7.6). Every element is computed before any
/// is stored, so `f = {q, f[0]}` reads the old `f`.
fn assign_concat_with_collections<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    dst: &ArrayRef<'ctx>,
    dst_shape: &ArrayShape<'d>,
    e: sv_lang::Expression<'d>,
    releases: bool,
) -> Result<(), LoweringError> {
    let refuse = |what: &str| {
        Err(LoweringError::unsupported_at(
            &format!("an unpacked array concatenation with {what}"),
            &cx.fn_sym,
        ))
    };
    // A class-handle element is owned by its row: every value stored carries
    // its own reference and the one the row held is given back.
    let owns = !crate::collections::handle_offsets(&dst_shape.elem).is_empty();
    // The target's elements are the sub-arrays of its first dimension
    // (`inner` rows each; 1 for a one-dimensional target).
    let inner = dst_shape.stride(0);
    let count = dst_shape.size(0);
    let i64t = cx.ctx.i64_type();
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let width = dst_shape.elem.value_width();
    enum Part<'ctx, 'd> {
        /// Elements known now, each `inner` row values (owned).
        Fixed(Vec<Vec<PackedValue<'ctx>>>),
        Coll(crate::collections::Coll<'d>, IntValue<'ctx>, IntValue<'ctx>),
    }
    let mut parts = Vec::new();
    let mut total = i64t.const_zero();
    for o in e.operands() {
        let ty = o.expr_type();
        if is_single_collection_element(&dst_shape.elem, ty.as_ref()) {
            // An item assignable to the collection element type is ONE
            // element (a copy), never spliced.
            if inner != 1 {
                return refuse("a collection element operand and a multi-dimensional target");
            }
            let v = element_value(cx, o, &dst_shape.elem, owns)?;
            total =
                cx.b.build_int_add(total, i64t.const_int(1, false), "cat_total")
                    .unwrap();
            parts.push(Part::Fixed(vec![vec![v]]));
        } else if ty
            .as_ref()
            .is_some_and(sv_sim_sim::state::storage::is_collection)
        {
            let Some(c) = crate::collections::coll_of_expr(&o) else {
                return refuse("a collection operand of an unsupported kind");
            };
            if inner > 1 && (c.width as u64) != inner * u64::from(width) {
                return refuse("an operand whose element shape differs from the target's");
            }
            let h = crate::collections::handle_of(cx, o)?;
            let n = crate::collections::size_of_handle(cx, &c, h);
            let n =
                cx.b.build_int_z_extend_or_bit_cast(n, i64t, "cat_n")
                    .unwrap();
            total = cx.b.build_int_add(total, n, "cat_total").unwrap();
            parts.push(Part::Coll(c, h, n));
        } else if ty.as_ref().is_some_and(|t| t.is_unpacked_array()) {
            let chunks: Vec<Vec<PackedValue<'ctx>>> = if inner == 1 {
                array_operand_elements_with(cx, o, &dst_shape.elem, owns)?
                    .into_iter()
                    .map(|v| vec![crate::select::fit_width(&op, v, width)])
                    .collect()
            } else {
                let (os, vs) = array_operand_rows(cx, o, &dst_shape.elem, owns)?;
                let vs: Vec<_> = vs
                    .into_iter()
                    .map(|v| crate::select::fit_width(&op, v, width))
                    .collect();
                let step = inner as usize;
                if os.dims.len() + 1 == dst_shape.dims.len() && vs.len() == step {
                    vec![vs]
                } else if os.dims.len() == dst_shape.dims.len() && vs.len() % step == 0 {
                    let mut cs: Vec<Vec<PackedValue<'ctx>>> =
                        vs.chunks(step).map(<[_]>::to_vec).collect();
                    if os.dims[0].0 > os.dims[0].1 {
                        cs.reverse();
                    }
                    cs
                } else {
                    return refuse(
                        "an operand whose rank or element shape differs from the target's",
                    );
                }
            };
            total =
                cx.b.build_int_add(
                    total,
                    i64t.const_int(chunks.len() as u64, false),
                    "cat_total",
                )
                .unwrap();
            parts.push(Part::Fixed(chunks));
        } else if inner == 1 {
            let v = element_value(cx, o, &dst_shape.elem, owns)?;
            total =
                cx.b.build_int_add(total, i64t.const_int(1, false), "cat_total")
                    .unwrap();
            parts.push(Part::Fixed(vec![vec![v]]));
        } else {
            return refuse("a non-array operand and a multi-dimensional target");
        }
    }
    let bad =
        cx.b.build_int_compare(
            IntPredicate::NE,
            total,
            i64t.const_int(count, false),
            "cat_bad",
        )
        .unwrap();
    let func =
        cx.b.get_insert_block()
            .and_then(|b| b.get_parent())
            .expect("in a function");
    let err_bb = cx.ctx.append_basic_block(func, "cat_size_err");
    let ok_bb = cx.ctx.append_basic_block(func, "cat_size_ok");
    let done_bb = cx.ctx.append_basic_block(func, "cat_size_done");
    cx.b.build_conditional_branch(bad, err_bb, ok_bb).unwrap();
    cx.b.position_at_end(err_bb);
    crate::stmt_shared::error_if(
        cx,
        cx.ctx.bool_type().const_int(1, false),
        e.source_range().start,
        &format!(
            "an unpacked array concatenation assigned to a fixed-size array of {count} elements has a different element count"
        ),
    )?;
    if owns {
        // The target keeps its value; the references the operand values were
        // given for it go back.
        for part in &parts {
            if let Part::Fixed(chunks) = part {
                for v in chunks.iter().flatten() {
                    crate::collections::release_handles(
                        cx,
                        &dst_shape.elem,
                        crate::expr::LoweredValue::Packed(*v),
                    );
                }
            }
        }
    }
    cx.b.build_unconditional_branch(done_bb).unwrap();
    cx.b.position_at_end(ok_bb);
    // Every fixed operand element is already an SSA value and a queue,
    // dynamic or associative array cannot alias `dst`, so each operand is
    // stored directly at its left-bound-first position `off + i` (linear in
    // the target size, so large targets need no unrolled per-position select
    // chain).
    let descending = dst_shape.dims[0].0 > dst_shape.dims[0].1;
    let row_of = |p: IntValue<'ctx>, j: u64| {
        let outer = if descending {
            cx.b.build_int_sub(i64t.const_int(count - 1, false), p, "cat_row")
                .unwrap()
        } else {
            p
        };
        if inner == 1 {
            outer
        } else {
            let base =
                cx.b.build_int_mul(outer, i64t.const_int(inner, false), "cat_row0")
                    .unwrap();
            cx.b.build_int_add(base, i64t.const_int(j, false), "cat_rowj")
                .unwrap()
        }
    };
    let put = |p: IntValue<'ctx>, j: u64, v: PackedValue<'ctx>| -> Result<(), LoweringError> {
        let row = row_of(p, j);
        if releases {
            release_row(cx, dst, row, &dst_shape.elem)?;
        }
        store_row(cx, dst, row, v)
    };
    let mut off = i64t.const_zero();
    for part in &parts {
        match part {
            Part::Fixed(chunks) => {
                for (i, chunk) in chunks.iter().enumerate() {
                    let p =
                        cx.b.build_int_add(off, i64t.const_int(i as u64, false), "cat_at")
                            .unwrap();
                    for (j, v) in chunk.iter().enumerate() {
                        put(p, j as u64, *v)?;
                    }
                }
                off =
                    cx.b.build_int_add(off, i64t.const_int(chunks.len() as u64, false), "cat_off")
                        .unwrap();
            }
            Part::Coll(c, h, n) => {
                // The size check passed, so `n` is at most `count`.
                let base = off;
                row_loop(cx, count, i64t.const_zero(), |j, acc| {
                    let live =
                        cx.b.build_int_compare(IntPredicate::ULT, j, *n, "cat_live")
                            .unwrap();
                    let func =
                        cx.b.get_insert_block()
                            .and_then(|b| b.get_parent())
                            .expect("in a function");
                    let st = cx.ctx.append_basic_block(func, "cat_st");
                    let nx = cx.ctx.append_basic_block(func, "cat_nx");
                    cx.b.build_conditional_branch(live, st, nx).unwrap();
                    cx.b.position_at_end(st);
                    let v = crate::collections::read_pos(cx, c, *h, j);
                    let v = crate::real::to_storage(
                        cx,
                        v,
                        c.width,
                        "a queue or dynamic array element wider than 128 bits",
                    )?;
                    let p = cx.b.build_int_add(base, j, "cat_at").unwrap();
                    for r in 0..inner {
                        let part = if inner == 1 {
                            crate::select::fit_width(&op, v, width)
                        } else {
                            crate::stream::bits(cx, v, (r as u32) * width, width)
                        };
                        // The collection keeps its reference; the row takes
                        // its own.
                        let part = crate::collections::retain_stored(cx, &dst_shape.elem, part);
                        put(p, r, part)?;
                    }
                    cx.b.build_unconditional_branch(nx).unwrap();
                    cx.b.position_at_end(nx);
                    Ok(acc)
                })?;
                off = cx.b.build_int_add(off, *n, "cat_off").unwrap();
            }
        }
    }
    cx.b.build_unconditional_branch(done_bb).unwrap();
    cx.b.position_at_end(done_bb);
    Ok(())
}

/// Copies the stored array `src` (shape `src_shape`) into `dst` (shape
/// `dst_shape`, the same dimension sizes), element by corresponding
/// position.
pub(crate) fn copy_array<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    dst: ArrayRef<'ctx>,
    dst_shape: &ArrayShape<'d>,
    src: ArrayRef<'ctx>,
    src_shape: &ArrayShape<'d>,
) -> Result<(), LoweringError> {
    // The source stays as it is (a callee's output formal is never released
    // at scope exit): `dst` takes its own references and gives back the
    // ones its old elements held.
    let releases = !crate::collections::handle_offsets(&dst_shape.elem).is_empty();
    emit_rows(
        cx,
        Some(&dst),
        dst_shape,
        src_shape,
        ArraySource::Stored(src),
        true,
        |k, v| {
            if releases {
                release_row(cx, &dst, k, &dst_shape.elem)?;
            }
            store_row(cx, &dst, k, v)
        },
    )
}

/// `dst <= rhs` for all or part of a module memory `dst`: every element's
/// new value is evaluated now and staged as a nonblocking write of its row.
pub(crate) fn nba_array<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    handle: PointerValue<'ctx>,
    dst: ArrayRef<'ctx>,
    dst_shape: &ArrayShape<'d>,
    rhs: sv_lang::Expression<'d>,
    delay: IntValue<'ctx>,
) -> Result<(), LoweringError> {
    let Backing::Memory(info) = dst.backing else {
        return Err(LoweringError::unsupported_at(
            "a nonblocking assignment to a local array",
            &cx.fn_sym,
        ));
    };
    let (src_shape, src) = array_operand(cx, rhs)?;
    // Staging copies each value, so the source need not be snapshotted. The
    // staged elements own references like any nonblocking store of a handle
    // (the elements they replace are not released, as for a scalar).
    let share = crate::func::user_callee(&peel_array_conversions(rhs)).is_none();
    nba_array_rows(
        cx, handle, dst, info, dst_shape, &src_shape, src, share, delay,
    )
}

/// Stages a nonblocking write of `value` to the row `row` (one element of a
/// module array) when `row.valid`.
pub(crate) fn nba_stage_ref<'ctx>(
    cx: &LowerCtx<'ctx, '_, '_>,
    handle: PointerValue<'ctx>,
    row: &ArrayRef<'ctx>,
    value: PackedValue<'ctx>,
    delay: IntValue<'ctx>,
) -> Result<(), LoweringError> {
    let Backing::Memory(info) = row.backing else {
        return Err(LoweringError::unsupported_at(
            "a nonblocking assignment to a local array",
            &cx.fn_sym,
        ));
    };
    guarded(cx, row.valid, || {
        crate::select::stage_nba_row(cx, handle, info, row.base, value, delay)
    })
}

/// `dst <= <captured local array>` (LRM 1800-2023 10.4.2, an intra-assignment
/// event control on a whole array): the rows of the frame temporary `src`
/// holding the right-hand side captured when the statement began, staged as
/// nonblocking writes of `dst`'s rows now.
pub(crate) fn nba_array_from_local<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    handle: PointerValue<'ctx>,
    dst: ArrayRef<'ctx>,
    dst_shape: &ArrayShape<'d>,
    src: PointerValue<'ctx>,
    src_shape: &ArrayShape<'d>,
    delay: IntValue<'ctx>,
) -> Result<(), LoweringError> {
    let Backing::Memory(info) = dst.backing else {
        return Err(LoweringError::unsupported_at(
            "a nonblocking assignment to a local array",
            &cx.fn_sym,
        ));
    };
    let src = ArraySource::Stored(ArrayRef::whole(cx, Backing::Local(src)));
    nba_array_rows(
        cx, handle, dst, info, dst_shape, src_shape, src, true, delay,
    )
}

#[allow(clippy::too_many_arguments)]
fn nba_array_rows<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    handle: PointerValue<'ctx>,
    dst: ArrayRef<'ctx>,
    info: MemorySignalInfo,
    dst_shape: &ArrayShape<'d>,
    src_shape: &ArrayShape<'d>,
    src: ArraySource<'ctx, 'd>,
    share: bool,
    delay: IntValue<'ctx>,
) -> Result<(), LoweringError> {
    emit_rows(cx, None, dst_shape, src_shape, src, share, |k, v| {
        guarded(cx, dst.valid, || {
            let row = cx.b.build_int_add(dst.base, k, "arr_row").unwrap();
            crate::select::stage_nba_row(cx, handle, info, row, v, delay)
        })
    })
}

/// `l == r`, `l != r`, `l === r` or `l !== r` on unpacked arrays (LRM
/// 1800-2023 §11.4.5, §7.6): the element comparisons at corresponding
/// positions, combined with a 4-state AND (any 0 makes the whole 0, else
/// any X makes it X); the inequalities negate that.
pub(crate) fn compare_arrays<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    e: sv_lang::Expression<'d>,
    bop: sv_lang::BinaryOp,
) -> Result<PackedValue<'ctx>, LoweringError> {
    use sv_lang::BinaryOp as B;
    let missing = || LoweringError::unsupported_at("array comparison operand", &cx.fn_sym);
    let (ls, lsrc) = array_operand(cx, e.left().ok_or_else(missing)?)?;
    let (rs, rsrc) = array_operand(cx, e.right().ok_or_else(missing)?)?;
    check_compatible(cx, &ls, &rs)?;
    let case = match bop {
        B::Equality | B::Inequality => false,
        B::CaseEquality | B::CaseInequality => true,
        _ => {
            return Err(LoweringError::unsupported_at(
                &format!("operator {bop:?} on unpacked arrays"),
                &cx.fn_sym,
            ));
        }
    };
    let op = OpBuilder {
        ctx: cx.ctx,
        b: &cx.b,
    };
    let i64t = cx.ctx.i64_type();
    let floating = ls.elem.is_floating();
    let bit = |llvm: IntValue<'ctx>| PackedValue {
        llvm,
        width: 1,
        repr: cx.repr,
        signed: false,
    };
    let cmp = |a: PackedValue<'ctx>, b: PackedValue<'ctx>| -> PackedValue<'ctx> {
        if floating {
            let (fa, fb) = (
                crate::real::bits_to_real(cx, a).llvm,
                crate::real::bits_to_real(cx, b).llvm,
            );
            let eq =
                cx.b.build_float_compare(inkwell::FloatPredicate::OEQ, fa, fb, "arr_feq")
                    .unwrap();
            op.pack(eq, cx.ctx.bool_type().const_zero(), 1, cx.repr, false)
        } else if case {
            op.caseeq4(a, b)
        } else {
            op.eq4(a, b)
        }
    };
    let one = op
        .pack(
            cx.ctx.bool_type().const_all_ones(),
            cx.ctx.bool_type().const_zero(),
            1,
            cx.repr,
            false,
        )
        .llvm;
    let combine = |acc: IntValue<'ctx>, c: PackedValue<'ctx>| op.and4(bit(acc), c).llvm;
    let result = match (lsrc, rsrc) {
        (ArraySource::Stored(lr), ArraySource::Stored(rr)) if same_row_order(&ls, &rs) => {
            row_loop(cx, ls.rows(), one, |k, acc| {
                let a = load_row(cx, &lr, k, &ls.elem)?;
                let b = load_row(cx, &rr, k, &rs.elem)?;
                Ok(combine(acc, cmp(a, b)))
            })?
        }
        (lsrc, rsrc) => {
            let row_values = |src: ArraySource<'ctx, 'd>,
                              shape: &ArrayShape<'d>|
             -> Result<Vec<PackedValue<'ctx>>, LoweringError> {
                match src {
                    ArraySource::Rows(exprs) => exprs
                        .iter()
                        .map(|&x| row_value(cx, x, &shape.elem, false))
                        .collect(),
                    ArraySource::Values(vs) => Ok(vs),
                    ArraySource::Stored(r) => (0..shape.rows())
                        .map(|k| load_row(cx, &r, i64t.const_int(k, false), &shape.elem))
                        .collect(),
                }
            };
            let lv = row_values(lsrc, &ls)?;
            let rv = row_values(rsrc, &rs)?;
            let mut acc = one;
            for (r, a) in lv.into_iter().enumerate() {
                let b = rv[corresponding_row(&ls, &rs, r as u64) as usize];
                acc = combine(acc, cmp(a, b));
            }
            acc
        }
    };
    let result = bit(result);
    Ok(match bop {
        B::Inequality | B::CaseInequality => op.not4(result),
        _ => result,
    })
}

/// `d[a:b] = rhs` / `d[a+:w] = rhs` where `d` is a dynamic array or queue
/// (LRM 1800-2023 7.6: an assignment whose left-hand side contains a slice
/// is a single assignment to the whole slice, element by element in
/// left-to-right order; the right-hand side is fully evaluated before any
/// element is written, so `q[0:1] = '{q[1], q[0]}` swaps). `None` when `lhs`
/// is not such a slice.
///
/// The right-hand side is an array of the slice's size (7.6) and the slice
/// must lie inside the collection (the LRM leaves an out-of-range slice
/// target undefined): otherwise a run-time error is issued and nothing is
/// written, the same policy as a size-mismatched fixed-array copy. Elements
/// that hold handles or reals are refused by name.
pub(crate) fn assign_coll_slice<'ctx, 'd>(
    cx: &LowerCtx<'ctx, 'd, '_>,
    lhs: sv_lang::Expression<'d>,
    rhs: sv_lang::Expression<'d>,
) -> Option<Result<(), LoweringError>> {
    if lhs.kind() != ExpressionKind::RangeSelect {
        return None;
    }
    let base = lhs.select_value()?;
    let c = crate::collections::coll_of_expr(&base)?;
    Some((|| {
        let unsupported = |what: &str| LoweringError::unsupported_at(what, &cx.fn_sym);
        if c.assoc {
            return Err(unsupported(
                "a slice of an associative array as an assignment target",
            ));
        }
        if c.elem.is_floating()
            || c.obj
            || c.leaky
            || !crate::collections::handle_offsets(&c.elem).is_empty()
        {
            return Err(unsupported(
                "a queue or dynamic-array slice assignment target with real or handle elements",
            ));
        }
        let from_coll = crate::collections::coll_of_expr(&rhs);
        let fixed = from_coll.is_none() && rhs.expr_type().is_some_and(|t| t.is_unpacked_array());
        if from_coll.is_none() && !fixed {
            return Err(unsupported(
                "a slice assignment whose right-hand side is not an array value",
            ));
        }
        let rc = from_coll.unwrap_or(c);
        let i64t = cx.ctx.i64_type();
        let one = i64t.const_int(1, false);
        let h = crate::collections::handle_of(cx, base)?;
        let (lo, hi) = crate::collections::slice_bounds(cx, &c, h, lhs)?;
        // The right-hand side first (a fresh copy, so it cannot alias).
        let new = if fixed {
            // The array's elements, in declaration order, as a fresh queue.
            let mut h = i64t.const_zero();
            for v in array_operand_elements(cx, rhs, &c.elem)? {
                let val =
                    crate::collections::elem_buf(cx, &c, crate::expr::LoweredValue::Packed(v))?;
                h = crate::collections::call(
                    cx,
                    &c,
                    sv_sim_runtime::collections_rt::CollOp::Push,
                    h,
                    crate::collections::Args {
                        val: Some(val),
                        ..Default::default()
                    },
                );
            }
            h
        } else {
            crate::collections::owned_value(cx, rhs)?
        };
        let n = crate::collections::size_of_handle(cx, &rc, new);
        let n =
            cx.b.build_int_z_extend_or_bit_cast(n, i64t, "slice_n")
                .unwrap();
        let size = crate::collections::size_of_handle(cx, &c, h);
        let size =
            cx.b.build_int_z_extend_or_bit_cast(size, i64t, "slice_dst_size")
                .unwrap();
        // Elements named by the slice: `hi - lo + 1`, or 0 when empty.
        let span =
            cx.b.build_int_add(
                cx.b.build_int_sub(hi, lo, "slice_span").unwrap(),
                one,
                "slice_cnt",
            )
            .unwrap();
        let empty =
            cx.b.build_int_compare(IntPredicate::SLT, span, i64t.const_zero(), "slice_empty")
                .unwrap();
        let count =
            cx.b.build_select(empty, i64t.const_zero(), span, "slice_count")
                .unwrap()
                .into_int_value();
        let mismatch =
            cx.b.build_int_compare(IntPredicate::NE, count, n, "slice_mismatch")
                .unwrap();
        let neg =
            cx.b.build_int_compare(IntPredicate::SLT, lo, i64t.const_zero(), "slice_neg")
                .unwrap();
        let past =
            cx.b.build_int_compare(IntPredicate::SGE, hi, size, "slice_past")
                .unwrap();
        let outside = cx.b.build_or(neg, past, "slice_outside").unwrap();
        let some =
            cx.b.build_int_compare(IntPredicate::UGT, n, i64t.const_zero(), "slice_some")
                .unwrap();
        let outside = cx.b.build_and(outside, some, "slice_outside_n").unwrap();
        let bad = cx.b.build_or(mismatch, outside, "slice_dst_bad").unwrap();
        let func =
            cx.b.get_insert_block()
                .and_then(|b| b.get_parent())
                .expect("in a function");
        let err_bb = cx.ctx.append_basic_block(func, "slice_err");
        let ok_bb = cx.ctx.append_basic_block(func, "slice_ok");
        let done_bb = cx.ctx.append_basic_block(func, "slice_done");
        cx.b.build_conditional_branch(bad, err_bb, ok_bb).unwrap();
        cx.b.position_at_end(err_bb);
        crate::stmt_shared::error_if(
            cx,
            cx.ctx.bool_type().const_int(1, false),
            lhs.source_range().start,
            "a queue or dynamic-array slice assigned to must lie inside the array and have the \
             size of the value assigned; nothing is assigned",
        )?;
        crate::collections::call(
            cx,
            &rc,
            sv_sim_runtime::collections_rt::CollOp::Release,
            new,
            crate::collections::Args::default(),
        );
        cx.b.build_unconditional_branch(done_bb).unwrap();
        cx.b.position_at_end(ok_bb);
        let op = OpBuilder {
            ctx: cx.ctx,
            b: &cx.b,
        };
        // Element k of the value into element lo + k of the slice; the
        // handle threads through the loop (a write may reallocate).
        let pre = cx.b.get_insert_block().expect("positioned");
        let head = cx.ctx.append_basic_block(func, "slice_loop");
        let body_bb = cx.ctx.append_basic_block(func, "slice_loop_row");
        let end_bb = cx.ctx.append_basic_block(func, "slice_loop_done");
        cx.b.build_unconditional_branch(head).unwrap();
        cx.b.position_at_end(head);
        let k = cx.b.build_phi(i64t, "k").unwrap();
        k.add_incoming(&[(&i64t.const_zero(), pre)]);
        let hp = cx.b.build_phi(i64t, "h").unwrap();
        hp.add_incoming(&[(&h, pre)]);
        let kv = k.as_basic_value().into_int_value();
        let hv = hp.as_basic_value().into_int_value();
        let more =
            cx.b.build_int_compare(IntPredicate::ULT, kv, n, "more")
                .unwrap();
        cx.b.build_conditional_branch(more, body_bb, end_bb)
            .unwrap();
        cx.b.position_at_end(body_bb);
        let v = crate::collections::read_at_coll(cx, &rc, new, kv)
            .into_packed("a slice element", &cx.fn_sym)?;
        let v = crate::select::fit_width(&op, v, c.width);
        let val = crate::collections::elem_buf(cx, &c, crate::expr::LoweredValue::Packed(v))?;
        let pos = cx.b.build_int_add(lo, kv, "slice_pos").unwrap();
        let nh = crate::collections::call(
            cx,
            &c,
            sv_sim_runtime::collections_rt::CollOp::Set,
            hv,
            crate::collections::Args {
                index: Some(pos),
                index2: Some(i64t.const_int(u64::from(c.queue), false)),
                val: Some(val),
                ..Default::default()
            },
        );
        let next = cx.b.build_int_add(kv, one, "k_next").unwrap();
        let latch = cx.b.get_insert_block().expect("positioned");
        k.add_incoming(&[(&next, latch)]);
        hp.add_incoming(&[(&nh, latch)]);
        cx.b.build_unconditional_branch(head).unwrap();
        cx.b.position_at_end(end_bb);
        crate::collections::set_handle(cx, base, hv)?;
        crate::collections::call(
            cx,
            &rc,
            sv_sim_runtime::collections_rt::CollOp::Release,
            new,
            crate::collections::Args::default(),
        );
        cx.b.build_unconditional_branch(done_bb).unwrap();
        cx.b.position_at_end(done_bb);
        Ok(())
    })())
}