cortiq_engine/

loader.rs

//! Weight loader: CMF tensor directory → Pipeline.
//!
//! Storage rule: models WITH task masks are dequantized to f32 (masked
//! execution needs f32 row access; skill files are small by design).
//! Models without masks keep quantized matrices zero-copy from the mmap
//! (`QTensor::Mapped`) — this is what lets a 15B file run in a few GB
//! of RSS instead of 60 GB of f32.
//!
//! Layer kinds come from `arch.layer_types`: FullAttention loads
//! `self_attn.*` (with auto-detected Qwen3.5 extras: per-head qk-norm by
//! tensor presence, output gate by q_proj row count); LinearAttention
//! loads the canonical core `vmf_attn.*` (folded at convert time).

use crate::kv_cache::LayerKvCache;
use crate::linear_core::{
    GdnCfg, GdnWeights, ShortConvCfg, ShortConvWeights, VmfPhaseCfg, VmfPhaseWeights,
};
use crate::pipeline::{
    AttnKind, DenseFfn, FfnKind, LayerWeights, MoeFfn, MtpModule, Pipeline, PipelineWeights,
};
use crate::qtensor::QTensor;
use crate::sampler::SamplerConfig;
use crate::tokenizer::Tokenizer;
use cortiq_core::quant::dequant_tensor;
use cortiq_core::{CmfError, CmfModel, LayerType, ModelArch};
use std::sync::Arc;

/// The refusal every generative entry point (`Pipeline::from_model*`, and
/// through it run/chat/route/bench) gives a file carrying the DECISION
/// feature bit. Such a file is served by the decision runtime instead.
pub const DECISION_MODEL_REFUSAL: &str =
    "this is a DECISION model; use `cortiq decide` or `cortiq serve`";

/// Tensor source selector (spec §9): backbone, one skill's overlay, or
/// a soft superposition of top-m skills (claim 14 working tensors).
pub enum Overlay<'a> {
    None,
    One(&'a str),
    /// (skill_id, weight); weights sum to 1 (softmax(−E/T) upstream).
    Blend(&'a [(String, f32)]),
}

impl Overlay<'_> {
    fn blend_touches(&self, model: &CmfModel, name: &str) -> bool {
        match self {
            Overlay::Blend(list) => list
                .iter()
                .any(|(sid, _)| model.tensor(&format!("skill.{sid}.{name}")).is_some()),
            _ => false,
        }
    }
}

fn dequant_by_name(model: &CmfModel, name: &str) -> Result<Vec<f32>, String> {
    let entry = model
        .tensor(name)
        .ok_or_else(|| format!("tensor '{name}' not found in CMF directory"))?;
    let mut out = vec![0.0f32; entry.n_elems()];
    dequant_tensor(entry, model.entry_bytes(entry), &mut out)?;
    Ok(out)
}

/// Weighted working tensor (claim 14): Σ wᵢ·Tᵢ, where Tᵢ is the
/// skill's replacement when present, else the backbone tensor.
fn blend_f32(model: &CmfModel, name: &str, list: &[(String, f32)]) -> Result<Vec<f32>, String> {
    let mut acc: Option<Vec<f32>> = None;
    for (sid, w) in list {
        let sname = format!("skill.{sid}.{name}");
        let src = if model.tensor(&sname).is_some() {
            &sname
        } else {
            name
        };
        let t = dequant_by_name(model, src)?;
        match &mut acc {
            None => {
                let mut t = t;
                for v in t.iter_mut() {
                    *v *= w;
                }
                acc = Some(t);
            }
            Some(a) => {
                for (av, tv) in a.iter_mut().zip(&t) {
                    *av += w * tv;
                }
            }
        }
    }
    acc.ok_or_else(|| "empty blend".into())
}

/// Dequantize a tensor fully into f32 (norms, masked models).
pub(crate) fn load_f32(model: &CmfModel, name: &str, ov: &Overlay) -> Result<Vec<f32>, String> {
    if ov.blend_touches(model, name) {
        if let Overlay::Blend(list) = ov {
            return blend_f32(model, name, list);
        }
    }
    let skill = match ov {
        Overlay::One(s) => Some(*s),
        _ => None,
    };
    let entry = model
        .resolve_tensor(name, skill)
        .ok_or_else(|| format!("tensor '{name}' not found in CMF directory"))?;
    let bytes = model.entry_bytes(entry);
    let mut out = vec![0.0f32; entry.n_elems()];
    dequant_tensor(entry, bytes, &mut out)?;
    Ok(out)
}

/// Build one layer's FFN (dense or MoE) under a given overlay. Shared
/// by the static loader AND dynamic per-token skill switching
/// (`Pipeline::set_active_skill`): switching skills = rebuilding the
/// FFN of the touched layers, cheap because Mapped tensors are just
/// re-resolved mmap pointers (no dequant, no copy).
pub(crate) fn build_layer_ffn(
    model: &Arc<CmfModel>,
    arch: &ModelArch,
    li: usize,
    force_f32: bool,
    ov: &Overlay,
) -> Result<FfnKind, CmfError> {
    build_ffn_at(model, arch, &format!("model.layers.{li}."), force_f32, ov)
}

/// The FFN under an arbitrary prefix. Split out of `build_layer_ffn` so the
/// MTP block can reuse it: Qwen3.6's MTP layer carries a full MoE mlp
/// (router + 256 experts + shared expert), not the dense one the head was
/// first written against.
pub(crate) fn build_ffn_at(
    model: &Arc<CmfModel>,
    arch: &ModelArch,
    prefix: &str,
    force_f32: bool,
    ov: &Overlay,
) -> Result<FfnKind, CmfError> {
    let prefix = prefix.to_string();
    let load_dense = |p: &str| -> Result<DenseFfn, CmfError> {
        let gate_proj = load_matrix(model, &format!("{p}gate_proj.weight"), force_f32, ov)?;
        let up_proj = load_matrix(model, &format!("{p}up_proj.weight"), force_f32, ov)?;
        let down_proj = load_matrix(model, &format!("{p}down_proj.weight"), force_f32, ov)?;
        // FFN triple invariant (holds for dense and each MoE expert;
        // enforced loudly so a malformed defrag/repack — spec §11 — fails
        // at load instead of silently mis-computing). inter' is per-layer.
        let inter = gate_proj.rows();
        if up_proj.rows() != inter || down_proj.cols() != inter {
            return Err(CmfError::Parse(format!(
                "{p}: FFN dims disagree (gate.rows={inter}, up.rows={}, \
                 down.cols={}); all three must equal inter'",
                up_proj.rows(),
                down_proj.cols()
            )));
        }
        if down_proj.rows() != arch.hidden_size {
            return Err(CmfError::Parse(format!(
                "{p}: down_proj.rows={} != hidden_size={}",
                down_proj.rows(),
                arch.hidden_size
            )));
        }
        // The transposed down, when the file carries one: the per-token
        // sparse path needs a neuron's down weights contiguous.
        let dt_name = format!("{p}down_proj.t.weight");
        let down_t = match model.tensor(&dt_name) {
            Some(_) => Some(load_matrix(model, &dt_name, force_f32, ov)?),
            None => None,
        };
        if let Some(t) = &down_t
            && (t.rows() != inter || t.cols() != arch.hidden_size)
        {
            return Err(CmfError::Parse(format!(
                "{p}down_proj.t: [{}, {}] != [{inter}, {}]",
                t.rows(),
                t.cols(),
                arch.hidden_size
            )));
        }
        // Task tubes (`…gate_proj.tube1.weight`, …): the neurons only
        // some tasks compute, each stored as its own triple so every
        // kernel runs it unchanged and an inactive tube's bytes are
        // never touched. Numbering is dense from 1; the first gap ends
        // the tube list.
        let mut segs = Vec::new();
        let mut start = inter;
        for k in 1.. {
            let gn = format!("{p}gate_proj.tube{k}.weight");
            if model.tensor(&gn).is_none() {
                break;
            }
            let gate = load_matrix(model, &gn, force_f32, ov)?;
            let up = load_matrix(model, &format!("{p}up_proj.tube{k}.weight"), force_f32, ov)?;
            let down = load_matrix(
                model,
                &format!("{p}down_proj.tube{k}.weight"),
                force_f32,
                ov,
            )?;
            let width = gate.rows();
            if up.rows() != width || down.cols() != width || down.rows() != arch.hidden_size {
                return Err(CmfError::Parse(format!(
                    "{p}tube{k}: dims disagree (gate.rows={width}, up.rows={}, \
                     down=[{}, {}], hidden={})",
                    up.rows(),
                    down.rows(),
                    down.cols(),
                    arch.hidden_size
                )));
            }
            segs.push(crate::pipeline::FfnSeg {
                gate,
                up,
                down,
                start,
                width,
            });
            start += width;
        }
        Ok(DenseFfn {
            gate_proj,
            up_proj,
            down_proj,
            act: crate::pipeline::Act::from_arch_full(arch),
            down_t,
            segs,
        })
    };
    let router_name = format!("{prefix}mlp.gate.weight");
    // Cortiq Embryo: resonance-routed experts carry NO gate tensor — the
    // MoE is keyed on the first expert + the arch flag (a grown genome
    // appends `mlp.experts.{E}.*` records without rewriting anything).
    let resonance_moe = model.tensor(&router_name).is_none()
        && arch.moe.as_ref().is_some_and(|m| m.router_resonance)
        && model
            .tensor(&format!("{prefix}mlp.experts.0.gate_proj.weight"))
            .is_some();
    if model.tensor(&router_name).is_none() && !resonance_moe {
        return Ok(FfnKind::Dense(load_dense(&format!("{prefix}mlp."))?));
    }
    let cfg = arch.moe.as_ref().ok_or_else(|| {
        CmfError::Parse(format!(
            "{router_name} present but header has no arch.moe block"
        ))
    })?;
    // Experts enumerate by TENSOR PRESENCE up to the header count — a
    // moe-defrag'd specialist keeps a per-layer contiguous prefix of
    // renumbered experts (fewer than arch.moe.num_experts), with the
    // router rows sliced to match.
    let mut experts = Vec::new();
    for e in 0..cfg.num_experts {
        if model
            .tensor(&format!("{prefix}mlp.experts.{e}.gate_proj.weight"))
            .is_none()
        {
            break;
        }
        experts.push(load_dense(&format!("{prefix}mlp.experts.{e}."))?);
    }
    if experts.is_empty() {
        return Err(CmfError::Parse(format!(
            "{prefix}: router present but no expert tensors"
        )));
    }
    // Growth records (`kind = "expert_append"`, spec §9.5.1): the experts
    // the records append to THIS layer, mounted behind the trunk experts
    // in header order. `arch.moe.num_experts` stays E0; the trunk count
    // is `e0`, the tail `experts[e0..]` is described by `grown`.
    let e0 = experts.len();
    let grown_loads = if resonance_moe {
        mount_growth(model, &prefix, ov, &load_dense)?
    } else {
        Vec::new()
    };
    let mut grown_desc: Vec<(Vec<f32>, Vec<f32>, f32, f32)> = Vec::new();
    let mut grown: Vec<crate::pipeline::GrownExpert> = Vec::new();
    for g in grown_loads {
        experts.push(g.ffn);
        grown_desc.push((g.mu, g.u, g.bias, g.shell));
        grown.push(g.meta);
    }
    let shared = if model
        .tensor(&format!("{prefix}mlp.shared_expert.gate_proj.weight"))
        .is_some()
    {
        let gate_name = format!("{prefix}mlp.shared_expert_gate.weight");
        Some((
            load_dense(&format!("{prefix}mlp.shared_expert."))?,
            if model.tensor(&gate_name).is_some() {
                Some(load_matrix(model, &gate_name, force_f32, ov)?)
            } else {
                None
            },
        ))
    } else {
        None
    };
    // LFM2-MoE selection bias (`mlp.expert_bias`): present iff the model
    // routes with a bias; loaded by tensor presence.
    let bias_name = format!("{prefix}mlp.expert_bias");
    let expert_bias = if model.tensor(&bias_name).is_some() {
        Some(load_f32(model, &bias_name, ov).map_err(CmfError::Parse)?)
    } else {
        None
    };
    // CMF_MOE_TOPK=N (opt-in): route to fewer experts than the header
    // asks. MoE decode is memory-bound — every selected expert streams
    // its three matrices per token — so halving k halves that traffic;
    // the renormalized top-k keeps the mixture a proper average.
    // Quality is the experiment — measure ppl before trusting.
    let top_k = std::env::var("CMF_MOE_TOPK")
        .ok()
        .and_then(|v| v.parse::<usize>().ok())
        .filter(|&k| k >= 1 && k <= cfg.top_k)
        .inspect(|k| tracing::info!("MoE top_k override: {} (header {})", k, cfg.top_k))
        .unwrap_or(cfg.top_k);
    // CMF_MOE_TAU=0.x (opt-in): adaptive routing — see MoeFfn::route_tau.
    let route_tau = std::env::var("CMF_MOE_TAU")
        .ok()
        .and_then(|v| v.parse::<f32>().ok())
        .filter(|&t| t > 0.0 && t < 1.0)
        .inspect(|t| tracing::info!("MoE adaptive routing: tau {t}"));
    let mask = moe_task_mask(model, &prefix, experts.len());
    let router = if resonance_moe {
        // never read (selection is by descriptors); zero placeholder in RAM
        QTensor::from_f32(
            vec![0.0; experts.len() * arch.hidden_size],
            experts.len(),
            arch.hidden_size,
        )
    } else {
        load_matrix(model, &router_name, force_f32, ov)?
    };
    if router.rows() != experts.len() {
        return Err(CmfError::Parse(format!(
            "{router_name}: {} rows != {} experts",
            router.rows(),
            experts.len()
        )));
    }
    let top_k = top_k.min(experts.len());
    // Gemma-4: per-expert weight scale after the top-k renorm; its
    // presence also marks the scale-less-rms router input (the folded
    // router gain — see the converter).
    let pes_name = format!("{prefix}mlp.per_expert_scale");
    let per_expert_scale = if model.tensor(&pes_name).is_some() {
        Some(load_f32(model, &pes_name, ov).map_err(CmfError::Parse)?)
    } else {
        None
    };
    let router_input_norm = per_expert_scale.is_some();
    // Cortiq Embryo resonance descriptors: per-expert records
    // `mlp.experts.{e}.desc.{mu,u,bias}` (append-only growth) or the legacy
    // per-layer `mlp.desc.{mu,u,bias}` [E, ...]. Present → the router is a
    // placeholder and selection is by reconstruction error.
    let per_expert = model
        .tensor(&format!("{prefix}mlp.experts.0.desc.mu"))
        .is_some();
    if !per_expert && !grown.is_empty() {
        return Err(CmfError::Parse(format!(
            "{prefix}: expert_append records need per-expert descriptors \
             (mlp.experts.0.desc.mu); the stacked mlp.desc.* form cannot be grown"
        )));
    }
    let resonance = if per_expert {
        // Trunk descriptors first (E0 rows), then the grown rows behind
        // them: `shell` is +inf on the trunk, the record's finite
        // `desc.shell` on a grown expert.
        let ne_d = e0;
        let hidden = arch.hidden_size;
        let mut mu = Vec::with_capacity(experts.len() * hidden);
        let mut u = Vec::new();
        let mut bias = Vec::with_capacity(experts.len());
        let mut k = 0usize;
        for e in 0..ne_d {
            let m = load_f32(model, &format!("{prefix}mlp.experts.{e}.desc.mu"), ov)
                .map_err(CmfError::Parse)?;
            if m.len() != hidden {
                return Err(CmfError::Parse(format!(
                    "{prefix}mlp.experts.{e}.desc.mu: {} != {hidden}",
                    m.len()
                )));
            }
            mu.extend_from_slice(&m);
            let un = format!("{prefix}mlp.experts.{e}.desc.u");
            if model.tensor(&un).is_some() {
                let ue = load_f32(model, &un, ov).map_err(CmfError::Parse)?;
                let ke = ue.len() / hidden.max(1);
                if e == 0 {
                    k = ke;
                }
                if ke != k {
                    return Err(CmfError::Parse(format!("{un}: rank {ke} != {k}")));
                }
                u.extend_from_slice(&ue);
            }
            let bn = format!("{prefix}mlp.experts.{e}.desc.bias");
            bias.push(if model.tensor(&bn).is_some() {
                load_f32(model, &bn, ov)
                    .map_err(CmfError::Parse)?
                    .first()
                    .copied()
                    .unwrap_or(0.0)
            } else {
                0.0
            });
        }
        let mut shell = vec![f32::INFINITY; ne_d];
        for (gm, gu, gb, gs) in &grown_desc {
            if gm.len() != hidden || gu.len() != k * hidden {
                return Err(CmfError::Parse(format!(
                    "{prefix}: grown expert descriptor {}×{} != trunk {hidden}×{k}",
                    gm.len(),
                    gu.len() / hidden.max(1)
                )));
            }
            mu.extend_from_slice(gm);
            u.extend_from_slice(gu);
            bias.push(*gb);
            shell.push(*gs);
        }
        Some(crate::pipeline::Resonance {
            mu,
            u,
            k,
            bias,
            shell,
        })
    } else if model.tensor(&format!("{prefix}mlp.desc.mu")).is_some() {
        let mu = load_f32(model, &format!("{prefix}mlp.desc.mu"), ov).map_err(CmfError::Parse)?;
        let ne_d = experts.len();
        let hidden = arch.hidden_size;
        if mu.len() != ne_d * hidden {
            return Err(CmfError::Parse(format!(
                "{prefix}mlp.desc.mu: {} != {ne_d}×{hidden}",
                mu.len()
            )));
        }
        let u_name = format!("{prefix}mlp.desc.u");
        let (u, k) = if model.tensor(&u_name).is_some() {
            let u = load_f32(model, &u_name, ov).map_err(CmfError::Parse)?;
            let k = u.len() / (ne_d * hidden).max(1);
            (u, k)
        } else {
            (Vec::new(), 0)
        };
        let b_name = format!("{prefix}mlp.desc.bias");
        let bias = if model.tensor(&b_name).is_some() {
            load_f32(model, &b_name, ov).map_err(CmfError::Parse)?
        } else {
            vec![0.0; ne_d]
        };
        Some(crate::pipeline::Resonance {
            mu,
            u,
            k,
            bias,
            shell: Vec::new(),
        })
    } else {
        None
    };
    let moe = MoeFfn {
        grown,
        router,
        experts,
        top_k,
        route_tau,
        norm_topk_prob: cfg.norm_topk_prob,
        router_sigmoid: cfg.router_sigmoid,
        expert_bias,
        routed_scaling: cfg.routed_scaling_factor.unwrap_or(1.0),
        shared,
        stats: std::cell::RefCell::new(Vec::new()),
        act_sq: std::cell::RefCell::new(Vec::new()),
        act_rows: std::cell::RefCell::new(Vec::new()),
        mask,
        per_expert_scale,
        router_input_norm,
        resonance,
    };
    // Gemma-4 dual-branch layer: a dense MLP coexists with the routed
    // experts, each branch inside its own norm sandwich.
    if model
        .tensor(&format!("{prefix}mlp.gate_proj.weight"))
        .is_some()
    {
        let norm = |suffix: &str| -> Result<Vec<f32>, CmfError> {
            load_f32(model, &format!("{prefix}{suffix}.weight"), ov).map_err(CmfError::Parse)
        };
        return Ok(FfnKind::DenseMoe(Box::new(crate::pipeline::DenseMoeFfn {
            dense: load_dense(&format!("{prefix}mlp."))?,
            moe,
            post_norm_1: norm("post_feedforward_layernorm_1")?,
            pre_norm_2: norm("pre_feedforward_layernorm_2")?,
            post_norm_2: norm("post_feedforward_layernorm_2")?,
        })));
    }
    Ok(FfnKind::Moe(moe))
}

// ───────────────────────── growth records ─────────────────────────

/// Which `expert_append` records the loader mounts (`CMF_GROWTH`).
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum GrowthMode {
    /// Records with `status == "active"` (the default).
    Active,
    /// Every record that is not retired (`CMF_GROWTH=all`: quarantine
    /// and stale_regate too — the gate measurement of a fresh record).
    All,
    /// None (`CMF_GROWTH=off`: the plain genome, F0's forward).
    Off,
}

impl GrowthMode {
    pub fn label(self) -> &'static str {
        match self {
            Self::Active => "active",
            Self::All => "all",
            Self::Off => "off",
        }
    }

    /// Does this mode mount a record of `status`?
    pub fn admits(self, status: Option<&str>) -> bool {
        match self {
            Self::Off => false,
            Self::Active => status == Some("active"),
            Self::All => status != Some("retired"),
        }
    }
}

/// `CMF_GROWTH` = `active` (unset) | `all` | `off`.
pub fn growth_mode() -> GrowthMode {
    match std::env::var("CMF_GROWTH")
        .ok()
        .as_deref()
        .map(|v| v.trim().to_ascii_lowercase())
        .as_deref()
    {
        Some("off") | Some("0") | Some("none") => GrowthMode::Off,
        Some("all") => GrowthMode::All,
        _ => GrowthMode::Active,
    }
}

/// The `expert_append` records the current [`growth_mode`] mounts, with
/// their positions in `header.skills` (the chain rule of the format is
/// evaluated at that position).
pub fn mounted_growth_records(
    header: &cortiq_core::CmfHeader,
) -> Vec<(usize, &cortiq_core::SkillRecord)> {
    let mode = growth_mode();
    header
        .skills
        .iter()
        .enumerate()
        .filter(|(_, s)| {
            s.kind.as_deref() == Some(cortiq_core::knowledge::skill_kind::EXPERT_APPEND)
                && mode.admits(s.status.as_deref())
        })
        .collect()
}

/// One grown expert as the loader read it from a record.
struct GrownLoad {
    ffn: DenseFfn,
    mu: Vec<f32>,
    u: Vec<f32>,
    bias: f32,
    shell: f32,
    meta: crate::pipeline::GrownExpert,
}

/// The layer index of a `model.layers.{l}.` prefix (None for the MTP
/// block and other prefixes — no growth record addresses those).
fn layer_of_prefix(prefix: &str) -> Option<usize> {
    prefix
        .strip_prefix("model.layers.")?
        .strip_suffix('.')?
        .parse()
        .ok()
}

/// The grown experts of the mounted `expert_append` records that grow the
/// layer `prefix` addresses, in header order, each read from the record's
/// own tensors (`skill.{id}.model.layers.{l}.mlp.experts.{e}.*`, the
/// names the format's layout plan gives for the record's position). One
/// log line per mounted record and layer.
fn mount_growth(
    model: &Arc<CmfModel>,
    prefix: &str,
    ov: &Overlay,
    load_dense: &dyn Fn(&str) -> Result<DenseFfn, CmfError>,
) -> Result<Vec<GrownLoad>, CmfError> {
    use cortiq_core::knowledge::expert_leaf;
    let Some(layer) = layer_of_prefix(prefix) else {
        return Ok(Vec::new());
    };
    let records = mounted_growth_records(&model.header);
    let mut out = Vec::new();
    for (at, rec) in records {
        if !rec.layers.contains(&layer) {
            continue;
        }
        let plan = cortiq_core::expert_append_layout(&model.header, &model.tensors, at, rec)
            .map_err(|e| CmfError::Parse(format!("{prefix}: growth record '{}': {e}", rec.id)))?;
        let mut by_expert: std::collections::BTreeMap<usize, Vec<&cortiq_core::ExpertTensorSpec>> =
            Default::default();
        for p in plan.iter().filter(|p| p.layer == layer) {
            by_expert.entry(p.expert).or_default().push(p);
        }
        let first = out.len();
        for (e, leaves) in &by_expert {
            let name = |leaf: &str| -> Result<String, CmfError> {
                leaves
                    .iter()
                    .find(|p| p.leaf == leaf)
                    .map(|p| p.name.clone())
                    .ok_or_else(|| {
                        CmfError::Parse(format!(
                            "{prefix}: growth record '{}' expert {e}: layout has no '{leaf}'",
                            rec.id
                        ))
                    })
            };
            let ffn = load_dense(&format!("skill.{}.{prefix}mlp.experts.{e}.", rec.id))?;
            let mu = load_f32(model, &name(expert_leaf::MU)?, ov).map_err(CmfError::Parse)?;
            let u = if leaves.iter().any(|p| p.leaf == expert_leaf::U) {
                load_f32(model, &name(expert_leaf::U)?, ov).map_err(CmfError::Parse)?
            } else {
                Vec::new()
            };
            let scalar = |leaf: &str| -> Result<f32, CmfError> {
                load_f32(model, &name(leaf)?, ov)
                    .map_err(CmfError::Parse)?
                    .first()
                    .copied()
                    .ok_or_else(|| {
                        CmfError::Parse(format!(
                            "{prefix}: growth record '{}' expert {e}: empty '{leaf}'",
                            rec.id
                        ))
                    })
            };
            let bias = scalar(expert_leaf::BIAS)?;
            let shell = scalar(expert_leaf::SHELL)?;
            if !shell.is_finite() {
                return Err(CmfError::Parse(format!(
                    "{prefix}: growth record '{}' expert {e}: desc.shell {shell} is not finite",
                    rec.id
                )));
            }
            out.push(GrownLoad {
                ffn,
                mu,
                u,
                bias,
                shell,
                meta: crate::pipeline::GrownExpert {
                    record: rec.id.clone(),
                    record_index: at,
                    layer,
                    expert: *e,
                },
            });
        }
        let declared: Vec<usize> = by_expert.keys().copied().collect();
        tracing::info!(
            "growth: layer {layer}: mounted expert_append '{}' (status {}) — {} experts, declared {:?}, shells {:?}",
            rec.id,
            rec.status.as_deref().unwrap_or("?"),
            by_expert.len(),
            declared,
            out[first..].iter().map(|g| g.shell).collect::<Vec<_>>()
        );
    }
    Ok(out)
}

/// Task mask over routed experts: DTG-MA applied to MoE.
///
/// This is deliberately diagnostic-only. A route-mass sidecar can look good
/// on a short perplexity window and still collapse during a long
/// autoregressive generation: excluding a low-mass expert changes the model,
/// and the changed prefix changes every later route. Consequently a nearby
/// `<model>.moe-mass.json` is NEVER adopted implicitly. `CMF_MOE_MASK=...`
/// remains an explicit experiment and accepts both weighted and legacy count
/// files; production acceleration must preserve the full router and treat a
/// non-resident winner as an exact cold completion.
pub(crate) fn moe_task_mask(
    _model: &std::sync::Arc<CmfModel>,
    prefix: &str,
    ne: usize,
) -> Option<Vec<bool>> {
    use std::sync::OnceLock;
    static CFG: OnceLock<Option<(std::collections::HashMap<usize, Vec<u64>>, f64)>> =
        OnceLock::new();
    let cfg = CFG.get_or_init(|| {
        let path = std::path::PathBuf::from(std::env::var("CMF_MOE_MASK").ok()?);
        let shown = path.display();
        let text = std::fs::read_to_string(&path)
            .map_err(|e| tracing::warn!("CMF_MOE_MASK: cannot read {shown}: {e}"))
            .ok()?;
        let map: std::collections::HashMap<String, Vec<u64>> = serde_json::from_str(&text)
            .map_err(|e| tracing::warn!("CMF_MOE_MASK: bad JSON in {shown}: {e}"))
            .ok()?;
        // Weighted dumps store normalized route mass in fixed-point units
        // and therefore have per-layer totals many orders above the old
        // top-k vote counts. The magnitude distinguishes the two explicit
        // diagnostic formats without relying on a special filename.
        let weighted = map
            .values()
            .any(|row| row.iter().copied().sum::<u64>() >= 1_000_000);
        let cover = std::env::var("CMF_MOE_MASK_COVER")
            .ok()
            .and_then(|v| v.parse::<f64>().ok())
            .filter(|&c| c > 0.0 && c <= 1.0)
            .unwrap_or(if weighted { 0.925 } else { 0.9 });
        tracing::info!("MoE task mask: {shown}, cover {cover}");
        Some((
            map.into_iter()
                .filter_map(|(k, v)| Some((k.parse::<usize>().ok()?, v)))
                .collect(),
            cover,
        ))
    });
    let (stats, cover) = cfg.as_ref()?;
    // The layer index rides in the tensor prefix ("model.layers.N.").
    let li: usize = prefix
        .split("layers.")
        .nth(1)?
        .split('.')
        .next()?
        .parse()
        .ok()?;
    let counts = stats.get(&li)?;
    if counts.len() != ne {
        tracing::warn!(
            "CMF_MOE_MASK: layer {li} has {} counts, model has {ne} experts — skipped",
            counts.len()
        );
        return None;
    }
    let total: u64 = counts.iter().sum();
    if total == 0 {
        return None;
    }
    let mut order: Vec<usize> = (0..ne).collect();
    order.sort_unstable_by_key(|&e| std::cmp::Reverse(counts[e]));
    let mut mask = vec![false; ne];
    let mut acc = 0u64;
    let mut kept = 0usize;
    for &e in &order {
        mask[e] = true;
        acc += counts[e];
        kept += 1;
        if (acc as f64) >= cover * (total as f64) {
            break;
        }
    }
    tracing::info!(
        "MoE task mask L{li}: {kept}/{ne} experts for {:.0}% mass",
        cover * 100.0
    );
    Some(mask)
}

pub(crate) fn load_matrix(
    model: &Arc<CmfModel>,
    name: &str,
    force_f32: bool,
    ov: &Overlay,
) -> Result<QTensor, CmfError> {
    // Claim 14: a blended working tensor is materialized in f32 and
    // held resident (the overlay-cache slot); single skills stay
    // zero-copy pointers into the mmap.
    if ov.blend_touches(model, name) {
        if let Overlay::Blend(list) = ov {
            let entry = model
                .tensor(name)
                .ok_or_else(|| CmfError::MissingTensor(name.to_string()))?;
            let data =
                blend_f32(model, name, list).map_err(|e| CmfError::Parse(format!("blend: {e}")))?;
            return Ok(QTensor::from_f32(data, entry.shape[0], entry.shape[1]));
        }
    }
    let skill = match ov {
        Overlay::One(s) => Some(*s),
        _ => None,
    };
    // Tensor-source indirection (spec §9): the skill's replacement is
    // read in place of the backbone tensor — either/or, never a sum.
    let name: &str = &match skill {
        Some(sid) if model.tensor(&format!("skill.{sid}.{name}")).is_some() => {
            format!("skill.{sid}.{name}")
        }
        _ => name.to_string(),
    };
    let err = |e: String| CmfError::Parse(format!("weight loading: {e}"));
    if force_f32 {
        let entry = model
            .tensor(name)
            .ok_or_else(|| CmfError::MissingTensor(name.to_string()))?;
        if entry.shape.len() != 2 {
            return Err(err(format!("'{name}' is not 2-D")));
        }
        let data = load_f32(model, name, &Overlay::None).map_err(err)?;
        Ok(QTensor::from_f32(data, entry.shape[0], entry.shape[1]))
    } else {
        QTensor::from_model(model, name).map_err(err)
    }
}

impl Pipeline {
    /// Build a runnable pipeline from an opened CMF model.
    pub fn from_model(
        model: &Arc<CmfModel>,
        sampler_config: SamplerConfig,
    ) -> Result<Self, CmfError> {
        Self::from_model_with_skill(model, sampler_config, None)
    }

    /// Same, with a skill overlaid (spec §9): every layer tensor is
    /// resolved through tensor-source indirection — the skill's
    /// full-shape replacement is read in place of the backbone tensor.
    /// No per-skill model is ever assembled: Mapped tensors are
    /// pointers into the one shared mmap.
    pub fn from_model_with_skill(
        model: &Arc<CmfModel>,
        sampler_config: SamplerConfig,
        skill: Option<&str>,
    ) -> Result<Self, CmfError> {
        match skill {
            Some(s) => Self::from_model_with_overlay(model, sampler_config, &Overlay::One(s)),
            None => Self::from_model_with_overlay(model, sampler_config, &Overlay::None),
        }
    }

    /// Soft superposition (claim 14): working tensors accumulated from
    /// the given (skill, weight) list — softmax(−E/T) upstream.
    pub fn from_model_with_blend(
        model: &Arc<CmfModel>,
        sampler_config: SamplerConfig,
        blend: &[(String, f32)],
    ) -> Result<Self, CmfError> {
        Self::from_model_with_overlay(model, sampler_config, &Overlay::Blend(blend))
    }

    fn skill_file_guard(model: &CmfModel) -> Result<(), CmfError> {
        // A decision profile (encoder + resonance skills) has no language
        // model in it at all; the generative pipeline must not try to
        // assemble one from its tensors.
        if model.required_features & cortiq_core::format::features::DECISION != 0 {
            return Err(CmfError::Parse(DECISION_MODEL_REFUSAL.into()));
        }
        // A standalone skill file carries a PARTIAL tensor set cut against
        // a base; running it would be half a network answering questions.
        if model.required_features & cortiq_core::format::features::SKILL_FILE != 0 {
            return Err(CmfError::Parse(
                "this file is a standalone SKILL, not a runnable model — attach it: \
                 cortiq skill apply <base.cmf> <this file> -o specialist.cmf"
                    .into(),
            ));
        }
        Ok(())
    }

    fn from_model_with_overlay(
        model: &Arc<CmfModel>,
        sampler_config: SamplerConfig,
        ov: &Overlay,
    ) -> Result<Self, CmfError> {
        // Refuse non-runnable files first, before any process-wide state
        // (the device cache directory, the graph verdict) is touched on
        // their behalf.
        Self::skill_file_guard(model)?;
        // Small device caches (probe verdicts, compiled pipelines) go
        // beside the model: it is a directory the caller demonstrably
        // writes to, which `std::env::temp_dir()` is not inside an
        // Android app sandbox.
        if let Some(dir) = model.path.parent() {
            crate::gpu::set_cache_dir(dir.to_path_buf());
        }
        // The token-graph refusal verdict lives in each pipeline
        // (`Pipeline::graph_refused`): a new pipeline starts clean and
        // loading one — a skill lane mid-traffic — never touches the
        // verdict of the others (R4/NF-2; this used to reset a
        // process-wide flag under running slots).
        let skill = match ov {
            Overlay::One(s) => Some(*s),
            _ => None,
        };
        if let Some(sid) = skill {
            let known = model.header.skills.iter().any(|s| s.id == sid)
                || model.skill_tensors(sid).next().is_some();
            if !known {
                return Err(CmfError::Parse(format!(
                    "skill '{sid}' not in this container (header.skills: {:?})",
                    model
                        .header
                        .skills
                        .iter()
                        .map(|s| &s.id)
                        .collect::<Vec<_>>()
                )));
            }
            tracing::info!(
                "skill '{sid}': {} replacement tensors overlaid",
                model.skill_tensors(sid).count()
            );
        }
        let arch = model.arch().clone();
        let err = |e: String| CmfError::Parse(format!("weight loading: {e}"));
        if let Some(heads) = &arch.attention_heads_per_layer {
            if heads.len() != arch.num_layers {
                return Err(CmfError::Parse(format!(
                    "arch.attention_heads_per_layer has {} entries, expected {}",
                    heads.len(),
                    arch.num_layers
                )));
            }
            if let Some((li, &nh)) = heads
                .iter()
                .enumerate()
                .find(|(_, nh)| **nh == 0 || **nh % arch.num_kv_heads != 0)
            {
                return Err(CmfError::Parse(format!(
                    "layer {li} has {nh} Q heads, which must be nonzero and divisible by {} KV heads",
                    arch.num_kv_heads
                )));
            }
        }
        if arch
            .layer_types
            .iter()
            .any(|t| matches!(t, LayerType::SlidingAttention))
            && arch.sliding_window.is_none()
        {
            return Err(CmfError::Parse(
                "model has SlidingAttention layers but no arch.sliding_window".into(),
            ));
        }

        // Masks × quantized mmap: only the HEAD-mask path needs f32
        // slices, so f32 is forced only when some mask actually restricts
        // attention heads. FFN masks run sparse directly on the quant
        // bytes (sparse_ffn_quant), and embed/lm_head are never masked.
        // The old condition forced f32 for ANY mask: a 4.17 B file whose
        // mask touched only the FFN dequantized to 16.7 GB at load and
        // ran the bare-f32 GEMMs — 0.0 tok/s on a laptop that swapped,
        // and a 30× crawl on a 48-core server. A mask with no head rows
        // costs nothing now.
        let heads_masked = model.masks.masks.iter().any(|m| {
            m.head_masks.iter().any(|row| {
                let mut bits = 0usize;
                for &b in row.iter() {
                    bits += b.count_ones() as usize;
                }
                !row.is_empty() && bits < arch.num_attention_heads
            })
        });
        let force_f32 = heads_masked; // attention only (head masks)

        // ── Tokenizer: embedded → sidecar → byte-level fallback ──
        let mut tokenizer = if let Some(vocab_bytes) = &model.vocab {
            Tokenizer::from_bytes(vocab_bytes)
                .map_err(|e| CmfError::Parse(format!("embedded tokenizer: {e}")))?
        } else if let Some(g) = &model.header.genome {
            // A genome binds its tokenizer (the trunk hash covers the VOCAB
            // section): whatever tokenizer.json lies beside the file is not
            // part of it (NF-5; core refuses such a file at open as well).
            return Err(CmfError::Parse(format!(
                "genome '{}' carries no embedded tokenizer (VOCAB) — a sidecar tokenizer.json \
                 is not part of the genome; refusing",
                g.id
            )));
        } else {
            let sidecar = model.path.with_file_name("tokenizer.json");
            if sidecar.exists() {
                Tokenizer::from_file(&sidecar)
                    .map_err(|e| CmfError::Parse(format!("sidecar tokenizer: {e}")))?
            } else {
                tracing::warn!("no tokenizer in file or sidecar — using byte-level fallback");
                Tokenizer::byte_level()
            }
        };
        // Chat/eos bundle (spec §6.1): the FILE defines chat behavior.
        if let Some(tc) = &model.header.tokenizer_config {
            tokenizer.chat_template = tc.chat_template.clone();
            tokenizer.extra_eos.extend(tc.eos_token_ids.iter().copied());
            if tokenizer.bos_token_id.is_none() {
                tokenizer.bos_token_id = tc.bos_token_id;
            }
            tracing::info!(
                "chat bundle: template {} chars, {} stop ids",
                tc.chat_template.as_deref().map(str::len).unwrap_or(0),
                tc.eos_token_ids.len()
            );
        }
        // Gemma's contract requires <bos> at sequence start, but its
        // tokenizer.json post-processor does not add it (the chat
        // template does). Raw prompts need it too — word salad without.
        if arch.arch_name.to_lowercase().contains("gemma") && tokenizer.bos_token_id.is_some() {
            tokenizer.add_bos = true;
        }

        // ── Top-level weights (never masked → always quantized) ──
        let embed_tokens = load_matrix(model, "model.embed_tokens.weight", false, ov)?;
        let final_norm = if arch.qwen4_exp.is_some() {
            // qwen4_exp folds its four streams through a learned global
            // hyper mixer and has no conventional final RMSNorm.
            vec![0.0; arch.hidden_size]
        } else {
            load_f32(model, "model.norm.weight", ov).map_err(err)?
        };
        let lm_head = if model.tensor("lm_head.weight").is_some() {
            load_matrix(model, "lm_head.weight", false, ov)?
        } else if arch.tie_word_embeddings {
            // Tied: reuse the embedding matrix (re-open, cheap for Mapped).
            load_matrix(model, "model.embed_tokens.weight", false, ov)?
        } else {
            return Err(CmfError::MissingTensor(
                "lm_head.weight (and tie_word_embeddings is false)".into(),
            ));
        };

        // ── Linear-core geometry (required if any linear layer exists) ──
        let has_linear = arch
            .layer_types
            .iter()
            .any(|t| matches!(t, LayerType::LinearAttention));
        let mut vmf_cfg = None;
        let mut gdn_cfg = None;
        // `Some` means this file uses the versioned Phase-Delta operator;
        // the sorted indices are threaded into each layer's weights below.
        // Legacy kinds must not carry selector metadata: accepting and
        // ignoring it would recreate the original silent semantic break.
        let mut phase_delta_layers: Option<Vec<usize>> = None;
        if !has_linear
            && arch
                .linear_core
                .as_ref()
                .is_some_and(|lc| lc.phase_delta_layers.is_some())
        {
            return Err(CmfError::Parse(
                "phase_delta_layers is only valid with LinearAttention and \
                 vmf_phase_delta_v1"
                    .into(),
            ));
        }
        if has_linear {
            let lc = arch.linear_core.as_ref().ok_or_else(|| {
                CmfError::Parse(
                    "model has LinearAttention layers but no arch.linear_core — \
                     reconvert with the current converter"
                        .into(),
                )
            })?;
            let need = |v: Option<usize>, name: &str| {
                v.ok_or_else(|| CmfError::Parse(format!("linear core needs arch.{name}")))
            };
            match lc.kind.as_str() {
                "vmf_phase" => {
                    if lc.phase_delta_layers.is_some() {
                        return Err(CmfError::Parse(
                            "legacy vmf_phase cannot carry phase_delta_layers; use vmf_phase_delta_v1"
                                .into(),
                        ));
                    }
                    vmf_cfg = Some(VmfPhaseCfg {
                        num_heads: lc.num_heads,
                        nphase: need(lc.nphase, "linear_core.nphase")?,
                        value_head_dim: lc.value_head_dim,
                        hidden_size: arch.hidden_size,
                        // Phase-mass correction: default 0 (disabled); CMF_PHASE_MASS
                        // widens the phase kernel for folded-unhealed models.
                        phase_mass: std::env::var("CMF_PHASE_MASS")
                            .ok()
                            .and_then(|v| v.parse().ok())
                            .unwrap_or(0.0),
                    });
                }
                "vmf_phase_delta_v1" => {
                    let mut selected = lc.phase_delta_layers.clone().ok_or_else(|| {
                        CmfError::Parse(
                            "vmf_phase_delta_v1 requires a non-empty phase_delta_layers selector"
                                .into(),
                        )
                    })?;
                    if selected.is_empty() {
                        return Err(CmfError::Parse(
                            "vmf_phase_delta_v1 phase_delta_layers is empty".into(),
                        ));
                    }
                    if arch.layer_types.len() != arch.num_layers {
                        return Err(CmfError::Parse(format!(
                            "phase_delta layer schedule has {} entries, expected {}",
                            arch.layer_types.len(),
                            arch.num_layers
                        )));
                    }
                    selected.sort_unstable();
                    for pair in selected.windows(2) {
                        if pair[0] == pair[1] {
                            return Err(CmfError::Parse(format!(
                                "phase_delta_layers contains duplicate layer {}",
                                pair[0]
                            )));
                        }
                    }
                    for &li in &selected {
                        if li >= arch.num_layers {
                            return Err(CmfError::Parse(format!(
                                "phase_delta layer {li} out of range for {} layers",
                                arch.num_layers
                            )));
                        }
                        if !matches!(arch.layer_types[li], LayerType::LinearAttention) {
                            return Err(CmfError::Parse(format!(
                                "phase_delta layer {li} is not a LinearAttention layer"
                            )));
                        }
                    }
                    let nphase = need(lc.nphase, "linear_core.nphase")?;
                    if lc.num_heads == 0 || nphase == 0 || lc.value_head_dim == 0 {
                        return Err(CmfError::Parse(
                            "vmf_phase_delta_v1 requires positive heads, nphase, and value_head_dim"
                                .into(),
                        ));
                    }
                    phase_delta_layers = Some(selected);
                    vmf_cfg = Some(VmfPhaseCfg {
                        num_heads: lc.num_heads,
                        nphase,
                        value_head_dim: lc.value_head_dim,
                        hidden_size: arch.hidden_size,
                        // Phase-Delta's normalized features are a closed
                        // operator contract. CMF_PHASE_MASS is a legacy
                        // tuning knob and is intentionally ignored by the
                        // phase_delta branch in linear_core.
                        phase_mass: 0.0,
                    });
                }
                "gated_delta_net" => {
                    if lc.phase_delta_layers.is_some() {
                        return Err(CmfError::Parse(
                            "gated_delta_net cannot carry phase_delta_layers".into(),
                        ));
                    }
                    gdn_cfg = Some(GdnCfg {
                        num_v_heads: lc.num_heads,
                        num_k_heads: need(arch.linear_num_key_heads, "linear_num_key_heads")?,
                        key_head_dim: need(arch.linear_key_head_dim, "linear_key_head_dim")?,
                        value_head_dim: lc.value_head_dim,
                        conv_kernel: need(arch.linear_conv_kernel_dim, "linear_conv_kernel_dim")?,
                        hidden_size: arch.hidden_size,
                        rms_eps: arch.rms_norm_eps,
                        output_gate_sigmoid: false,
                    });
                }
                other => {
                    return Err(CmfError::Parse(format!(
                        "unknown linear core '{other}' (this runtime executes: \
                         gated_delta_net, vmf_phase, vmf_phase_delta_v1)"
                    )));
                }
            }
        }

        // ── KDA geometry (Kimi Linear / Kimi-K3 delta-attention layers) ──
        let has_kda = arch.layer_types.iter().any(|t| matches!(t, LayerType::Kda));
        let kda_cfg = if has_kda {
            let need = |v: Option<usize>, name: &str| {
                v.ok_or_else(|| CmfError::Parse(format!("KDA core needs arch.{name}")))
            };
            Some(crate::linear_core::KdaCfg {
                num_heads: need(arch.linear_num_key_heads, "linear_num_key_heads")?,
                head_k_dim: need(arch.linear_key_head_dim, "linear_key_head_dim")?,
                head_v_dim: need(arch.linear_value_head_dim, "linear_value_head_dim")?,
                conv_kernel: need(arch.linear_conv_kernel_dim, "linear_conv_kernel_dim")?,
                hidden_size: arch.hidden_size,
                rms_eps: arch.rms_norm_eps,
            })
        } else {
            None
        };

        // ── Short-convolution geometry (LFM2 conv mixer layers) ──
        let has_short_conv = arch
            .layer_types
            .iter()
            .any(|t| matches!(t, LayerType::ShortConv));
        let short_conv_cfg = if has_short_conv {
            Some(ShortConvCfg {
                hidden_size: arch.hidden_size,
                kernel: arch.linear_conv_kernel_dim.ok_or_else(|| {
                    CmfError::Parse(
                        "model has ShortConv layers but no arch.linear_conv_kernel_dim — \
                         reconvert with the current converter"
                            .into(),
                    )
                })?,
            })
        } else {
            None
        };

        // ── Layers ──
        let load_full_attn = |prefix: &str, layer: Option<usize>| -> Result<AttnKind, CmfError> {
            let t = |suffix: &str| load_matrix(model, &format!("{prefix}{suffix}"), force_f32, ov);
            let n = |suffix: &str| -> Option<Vec<f32>> {
                model
                    .tensor(&format!("{prefix}{suffix}"))
                    .and_then(|_| load_f32(model, &format!("{prefix}{suffix}"), ov).ok())
            };
            // DeepSeek-V2 MLA: the latent projections replace the k/v pair.
            if let Some(mla) = arch.mla.as_ref() {
                // Compressed q (K3/V3): q_a → rms → q_b; direct otherwise.
                let (q_proj, q_a, q_a_norm) = if mla.q_lora_rank.is_some() {
                    (
                        t("self_attn.q_b_proj.weight")?,
                        Some(t("self_attn.q_a_proj.weight")?),
                        Some(n("self_attn.q_a_layernorm.weight").ok_or_else(|| {
                            CmfError::Parse(format!("{prefix}: MLA needs q_a_layernorm"))
                        })?),
                    )
                } else {
                    (t("self_attn.q_proj.weight")?, None, None)
                };
                let hd = mla.qk_rope_head_dim + mla.qk_nope_head_dim;
                let nh = q_proj.rows() / hd;
                // YaRN mscale²: DeepSeek corrects the softmax scale by
                // (0.1·mscale_all_dim·ln(factor)+1)².
                let mut scale = 1.0 / (hd as f32).sqrt();
                if let Some(y) = arch.yarn.as_ref() {
                    if let Some(m) = y.mscale_all_dim.filter(|&m| m > 0.0) {
                        let ms = 0.1 * m * y.factor.ln() + 1.0;
                        scale *= ms * ms;
                    }
                }
                return Ok(AttnKind::Mla(Box::new(crate::pipeline::MlaWeights {
                    q_proj,
                    q_a,
                    q_a_norm,
                    kv_a: t("self_attn.kv_a_proj_with_mqa.weight")?,
                    kv_a_norm: n("self_attn.kv_a_layernorm.weight").ok_or_else(|| {
                        CmfError::Parse(format!("{prefix}: MLA needs kv_a_layernorm"))
                    })?,
                    kv_b: t("self_attn.kv_b_proj.weight")?,
                    o_proj: t("self_attn.o_proj.weight")?,
                    nh,
                    qk_rope: mla.qk_rope_head_dim,
                    qk_nope: mla.qk_nope_head_dim,
                    v_dim: mla.v_head_dim,
                    lora: mla.kv_lora_rank,
                    scale,
                    nope: mla.nope,
                })));
            }
            let wq = t("self_attn.q_proj.weight")?;
            let nh = layer
                .and_then(|li| {
                    arch.attention_heads_per_layer
                        .as_ref()
                        .and_then(|v| v.get(li).copied())
                })
                .unwrap_or(arch.num_attention_heads);
            // Qwen3.5 output gate: q_proj rows = 2·nh·hd (per-head [q; gate]).
            // Gemma-4 global layers legitimately have nh·global_head_dim
            // rows (which can equal 2·nh·hd) — never gated.
            let output_gate = arch.global_head_dim.is_none() && wq.rows() == 2 * nh * arch.head_dim;
            // Gemma-4 global layers run MQA at global_head_dim — their
            // q_proj legitimately carries nh·ghd rows.
            let is_global_layer = arch.global_head_dim.is_some()
                && layer.is_some_and(|li| {
                    arch.sliding_window_pattern
                        .is_some_and(|p| p > 0 && (li + 1) % p == 0)
                });
            let expect = if is_global_layer {
                nh * arch.global_head_dim.unwrap_or(arch.head_dim)
            } else {
                nh * arch.head_dim
            };
            if !output_gate && wq.rows() != expect {
                return Err(CmfError::Parse(format!(
                    "{prefix}self_attn.q_proj.weight rows={} != heads({nh}) * head_dim({})",
                    wq.rows(),
                    expect / nh.max(1)
                )));
            }
            let gate_name = format!("{prefix}self_attn.g_proj.weight");
            let softplus_gate = if model.tensor(&gate_name).is_some() {
                let gate = load_matrix(model, &gate_name, force_f32, ov)?;
                if gate.cols() != arch.hidden_size {
                    return Err(CmfError::Parse(format!(
                        "{gate_name} cols={} != hidden_size ({})",
                        gate.cols(),
                        arch.hidden_size
                    )));
                }
                let per_head = if gate.rows() == nh {
                    true
                } else if gate.rows() == nh * arch.head_dim {
                    false
                } else {
                    return Err(CmfError::Parse(format!(
                        "{gate_name} rows={} must equal heads ({nh}) or heads*head_dim ({})",
                        gate.rows(),
                        nh * arch.head_dim
                    )));
                };
                Some((gate, per_head))
            } else {
                None
            };
            // Qwen2-family projection biases (by tensor presence).
            let bias = match (
                n("self_attn.q_proj.bias"),
                n("self_attn.k_proj.bias"),
                n("self_attn.v_proj.bias"),
            ) {
                (Some(a), Some(b), Some(c)) => Some((a, b, c)),
                _ => None,
            };
            let wk = t("self_attn.k_proj.weight")?;
            let wv = t("self_attn.v_proj.weight")?;
            let wo = t("self_attn.o_proj.weight")?;
            // K/V/O geometry, checked here and never at run time: a
            // mapped matvec accepts any `out`/`x` at least as long as its
            // rows/cols, so a V narrower than the cache's head or an
            // o_proj wider than nh·v would otherwise run silently wrong.
            // KV heads come from the header — per layer when the model
            // varies them (MiMo-V2: 4 full / 8 sliding), Gemma-4 global
            // layers carry their own (heads, width).
            let (nkv_l, hd_l) = if is_global_layer {
                (
                    arch.num_global_kv_heads.unwrap_or(arch.num_kv_heads),
                    arch.global_head_dim.unwrap_or(arch.head_dim),
                )
            } else {
                (
                    layer
                        .and_then(|li| {
                            arch.kv_heads_per_layer
                                .as_ref()
                                .and_then(|v| v.get(li).copied())
                        })
                        .unwrap_or(arch.num_kv_heads),
                    arch.head_dim,
                )
            };
            let vd = if is_global_layer {
                hd_l
            } else {
                arch.v_head_dim.unwrap_or(hd_l)
            };
            let shape_err = |what: String| {
                CmfError::Parse(format!(
                    "{prefix}self_attn: {what} (heads {nh}, kv heads {nkv_l}, head_dim {hd_l}, \
                     v_head_dim {vd}) — file and header disagree"
                ))
            };
            if nkv_l == 0 || nh % nkv_l != 0 {
                return Err(shape_err(format!(
                    "{nkv_l} KV heads must be nonzero and divide {nh} Q heads"
                )));
            }
            if vd == 0 || vd > hd_l {
                return Err(shape_err(format!("v_head_dim {vd} must be in 1..={hd_l}")));
            }
            if wk.rows() != nkv_l * hd_l {
                return Err(shape_err(format!(
                    "k_proj rows={} != kv_heads*head_dim={}",
                    wk.rows(),
                    nkv_l * hd_l
                )));
            }
            if wv.rows() != nkv_l * vd {
                return Err(shape_err(format!(
                    "v_proj rows={} != kv_heads*v_head_dim={}",
                    wv.rows(),
                    nkv_l * vd
                )));
            }
            if wo.cols() != nh * vd {
                return Err(shape_err(format!(
                    "o_proj cols={} != heads*v_head_dim={}",
                    wo.cols(),
                    nh * vd
                )));
            }
            if vd < hd_l && (output_gate || softplus_gate.is_some()) {
                return Err(shape_err(
                    "an attention output gate with V heads narrower than Q/K is not supported"
                        .into(),
                ));
            }
            Ok(AttnKind::Full {
                wq,
                wk,
                wv,
                wo,
                q_norm: n("self_attn.q_norm.weight"),
                k_norm: n("self_attn.k_norm.weight"),
                output_gate,
                softplus_gate,
                bias,
            })
        };

        let load_linear_attn = |prefix: &str, li: usize| -> Result<AttnKind, CmfError> {
            if gdn_cfg.is_some() {
                // Faithful vendor operator: tensor names 1:1 with the source.
                let t = |suffix: &str| {
                    load_matrix(
                        model,
                        &format!("{prefix}linear_attn.{suffix}"),
                        force_f32,
                        ov,
                    )
                };
                let f = |suffix: &str| {
                    load_f32(model, &format!("{prefix}linear_attn.{suffix}"), ov).map_err(err)
                };
                return Ok(AttnKind::LinearGdn(GdnWeights {
                    in_proj_qkv: t("in_proj_qkv.weight")?,
                    in_proj_z: t("in_proj_z.weight")?,
                    in_proj_a: t("in_proj_a.weight")?,
                    in_proj_b: t("in_proj_b.weight")?,
                    conv1d: f("conv1d.weight")?,
                    a_log: f("A_log")?,
                    dt_bias: f("dt_bias")?,
                    norm: f("norm.weight")?,
                    out_proj: t("out_proj.weight")?,
                }));
            }
            let t = |suffix: &str| {
                load_matrix(model, &format!("{prefix}vmf_attn.{suffix}"), force_f32, ov)
            };
            let a_log = load_f32(model, &format!("{prefix}vmf_attn.A_log"), ov).map_err(err)?;
            // Selective-write gate κ (hybrid_k core): optional by tensor
            // presence — files without it run the classic phase kernel
            // bit-identically.
            let k_gate = if model
                .tensor(&format!("{prefix}vmf_attn.k_gate.weight"))
                .is_some()
            {
                Some((
                    t("k_gate.weight")?,
                    load_f32(model, &format!("{prefix}vmf_attn.k_gate.bias"), ov).map_err(err)?,
                ))
            } else {
                None
            };
            // Optional short causal conv (embryo genomes born with
            // --conv-k): `[hidden, 1, k]` taps, flattened.
            let conv = if model
                .tensor(&format!("{prefix}vmf_attn.conv1d.weight"))
                .is_some()
            {
                Some(load_f32(model, &format!("{prefix}vmf_attn.conv1d.weight"), ov).map_err(err)?)
            } else {
                None
            };
            let thq = t("thq.weight")?;
            let thk = t("thk.weight")?;
            let v_proj = t("v_proj.weight")?;
            let out_proj = t("out_proj.weight")?;
            let phase_delta = phase_delta_layers
                .as_ref()
                .is_some_and(|layers| layers.binary_search(&li).is_ok());
            if phase_delta {
                let cfg = vmf_cfg.ok_or_else(|| {
                    CmfError::Parse("phase_delta layer has no VMF geometry".into())
                })?;
                let expect = |name: &str, got: (usize, usize), want: (usize, usize)| {
                    if got != want {
                        Err(CmfError::Parse(format!(
                            "phase_delta {name} geometry [{}, {}] != [{}, {}]",
                            got.0, got.1, want.0, want.1
                        )))
                    } else {
                        Ok(())
                    }
                };
                expect(
                    "thq",
                    (thq.rows(), thq.cols()),
                    (cfg.num_heads * cfg.nphase, cfg.hidden_size),
                )?;
                expect(
                    "thk",
                    (thk.rows(), thk.cols()),
                    (cfg.num_heads * cfg.nphase, cfg.hidden_size),
                )?;
                expect(
                    "v_proj",
                    (v_proj.rows(), v_proj.cols()),
                    (cfg.num_heads * cfg.value_head_dim, cfg.hidden_size),
                )?;
                expect(
                    "out_proj",
                    (out_proj.rows(), out_proj.cols()),
                    (cfg.hidden_size, cfg.num_heads * cfg.value_head_dim),
                )?;
                if a_log.len() != cfg.num_heads * 2 * cfg.nphase {
                    return Err(CmfError::Parse(format!(
                        "phase_delta A_log length {} != {}",
                        a_log.len(),
                        cfg.num_heads * 2 * cfg.nphase
                    )));
                }
                if let Some((kw, kb)) = &k_gate {
                    expect(
                        "k_gate",
                        (kw.rows(), kw.cols()),
                        (cfg.num_heads, cfg.hidden_size),
                    )?;
                    if kb.len() != cfg.num_heads {
                        return Err(CmfError::Parse(format!(
                            "phase_delta k_gate.bias length {} != {}",
                            kb.len(),
                            cfg.num_heads
                        )));
                    }
                }
                if let Some(c) = &conv {
                    let n = model
                        .tensor(&format!("{prefix}vmf_attn.conv1d.weight"))
                        .ok_or_else(|| {
                            CmfError::Parse(
                                "phase_delta conv tensor disappeared during load".into(),
                            )
                        })?;
                    if n.shape.len() != 3
                        || n.shape[0] != cfg.hidden_size
                        || n.shape[1] != 1
                        || n.shape[2] < 2
                        || c.len() != n.shape.iter().product::<usize>()
                    {
                        return Err(CmfError::Parse(format!(
                            "phase_delta conv geometry {:?} / {} bytes is incompatible",
                            n.shape,
                            c.len()
                        )));
                    }
                }
            }
            Ok(AttnKind::Linear(VmfPhaseWeights {
                thq,
                conv,
                thk,
                v_proj,
                out_proj,
                decay: a_log.iter().map(|&a| (-(a as f64).exp()).exp()).collect(),
                k_gate,
                phase_delta,
            }))
        };

        // LFM2 short-conv mixer: in_proj [3·hidden, hidden], a depthwise
        // conv (stored f16 as `[hidden, 1, kernel]` → flattened taps), and
        // out_proj [hidden, hidden]. Names canonicalized at convert time.
        let load_short_conv = |prefix: &str| -> Result<AttnKind, CmfError> {
            let t = |suffix: &str| {
                load_matrix(
                    model,
                    &format!("{prefix}short_conv.{suffix}"),
                    force_f32,
                    ov,
                )
            };
            Ok(AttnKind::ShortConv(ShortConvWeights {
                in_proj: t("in_proj.weight")?,
                conv: load_f32(model, &format!("{prefix}short_conv.conv.weight"), ov)
                    .map_err(err)?,
                out_proj: t("out_proj.weight")?,
            }))
        };

        // KDA layer (Kimi Linear / Kimi-K3): faithful vendor tensors under
        // the `kda_attn.` canonical prefix. The output gate is full-rank
        // (g_proj, K3) or low-rank (g_a/g_b, Kimi-Linear-48B) by presence.
        let load_kda = |prefix: &str| -> Result<AttnKind, CmfError> {
            let t = |suffix: &str| {
                load_matrix(model, &format!("{prefix}kda_attn.{suffix}"), force_f32, ov)
            };
            let f = |suffix: &str| {
                load_f32(model, &format!("{prefix}kda_attn.{suffix}"), ov).map_err(err)
            };
            let gate = if model
                .tensor(&format!("{prefix}kda_attn.g_proj.weight"))
                .is_some()
            {
                crate::linear_core::KdaOutGate::Full(t("g_proj.weight")?)
            } else {
                crate::linear_core::KdaOutGate::LowRank(
                    t("g_a_proj.weight")?,
                    t("g_b_proj.weight")?,
                )
            };
            Ok(AttnKind::Kda(Box::new(crate::linear_core::KdaWeights {
                q_proj: t("q_proj.weight")?,
                k_proj: t("k_proj.weight")?,
                v_proj: t("v_proj.weight")?,
                conv_q: f("q_conv1d.weight")?,
                conv_k: f("k_conv1d.weight")?,
                conv_v: f("v_conv1d.weight")?,
                f_a: t("f_a_proj.weight")?,
                f_b: t("f_b_proj.weight")?,
                dt_bias: f("dt_bias")?,
                a_log: f("A_log")?,
                b_proj: t("b_proj.weight")?,
                gate,
                o_norm: f("o_norm.weight")?,
                o_proj: t("o_proj.weight")?,
                gate_lower_bound: arch.kda_gate_lower_bound.map(|v| v as f32),
            })))
        };

        // Natively bounded anchor (Embryo-O1 `swa_sink_v1`): the plain
        // q/k/v/o projections plus the TRAINED sink vectors
        // `self_attn.sink_k/sink_v.weight [kvh, S, hd]` (f32, weights —
        // not positions). The operator carries no qk-norm, gate or bias:
        // their presence is a contract violation, not something to
        // silently execute differently from the trainer.
        let load_bounded_attn = |prefix: &str, li: usize| -> Result<AttnKind, CmfError> {
            let ac = arch.anchor_core.as_ref().ok_or_else(|| {
                CmfError::Parse(format!(
                    "layer {li} is BoundedAttention but the header carries no anchor_core"
                ))
            })?;
            let t = |suffix: &str| load_matrix(model, &format!("{prefix}{suffix}"), force_f32, ov);
            let (nkv, hd, nh) = (arch.num_kv_heads, arch.head_dim, arch.num_attention_heads);
            for extra in [
                "self_attn.q_norm.weight",
                "self_attn.k_norm.weight",
                "self_attn.g_proj.weight",
                "self_attn.q_proj.bias",
                "self_attn.k_proj.bias",
                "self_attn.v_proj.bias",
            ] {
                if model.tensor(&format!("{prefix}{extra}")).is_some() {
                    return Err(CmfError::Parse(format!(
                        "{prefix}{extra}: the bounded anchor operator carries no qk-norm, \
                         gate or bias (docs/EMBRYO_BOUNDED_ANCHOR.md §1)"
                    )));
                }
            }
            let sink = |name: &str| -> Result<Vec<f32>, CmfError> {
                let full = format!("{prefix}self_attn.{name}.weight");
                let e = model.tensor(&full).ok_or_else(|| {
                    CmfError::MissingTensor(format!(
                        "{full} (a bounded anchor needs its trained sinks)"
                    ))
                })?;
                if e.shape.len() != 3
                    || e.shape[0] != nkv
                    || e.shape[1] != ac.sink
                    || e.shape[2] != hd
                {
                    return Err(CmfError::Parse(format!(
                        "{full}: shape {:?} != [{nkv}, {}, {hd}] (num_kv_heads, anchor_core.sink, head_dim)",
                        e.shape, ac.sink
                    )));
                }
                load_f32(model, &full, ov).map_err(err)
            };
            let wq = t("self_attn.q_proj.weight")?;
            let wk = t("self_attn.k_proj.weight")?;
            let wv = t("self_attn.v_proj.weight")?;
            let wo = t("self_attn.o_proj.weight")?;
            let expect = |name: &str, got: (usize, usize), want: (usize, usize)| {
                if got != want {
                    Err(CmfError::Parse(format!(
                        "{prefix}self_attn.{name}.weight is {}x{}, expected {}x{}",
                        got.0, got.1, want.0, want.1
                    )))
                } else {
                    Ok(())
                }
            };
            expect("q_proj", (wq.rows(), wq.cols()), (nh * hd, arch.hidden_size))?;
            expect("k_proj", (wk.rows(), wk.cols()), (nkv * hd, arch.hidden_size))?;
            expect("v_proj", (wv.rows(), wv.cols()), (nkv * hd, arch.hidden_size))?;
            expect("o_proj", (wo.rows(), wo.cols()), (arch.hidden_size, nh * hd))?;
            Ok(AttnKind::Bounded(Box::new(crate::bounded::BoundedWeights {
                wq,
                wk,
                wv,
                wo,
                sink_k: sink("sink_k")?,
                sink_v: sink("sink_v")?,
                sink: ac.sink,
                window: ac.window,
            })))
        };

        fn anyhow_like(ok: bool) -> Result<(), ()> {
            if ok { Ok(()) } else { Err(()) }
        }
        let mut layers = Vec::with_capacity(arch.num_layers);
        let is_g3n = arch.g3n.is_some();
        // Architectures that load their own layer stack below. DeepSeek-V4
        // has none of the canonical projections — no q/k/v/o_proj, no
        // per-layer gate_proj — so the generic loop would demand
        // `self_attn.q_proj.weight` and fail before its own loader ever ran.
        let owns_its_layers = is_g3n
            || arch.arch_name == "deepseek_v4"
            || arch.arch_name == "deepseek_v41"
            || arch.qwen4_exp.is_some();
        for li in 0..(if owns_its_layers { 0 } else { arch.num_layers }) {
            let prefix = format!("model.layers.{li}.");
            let attn = match arch.layer_types.get(li) {
                Some(LayerType::LinearAttention) => load_linear_attn(&prefix, li)?,
                Some(LayerType::Kda) => load_kda(&prefix)?,
                Some(LayerType::ShortConv) => load_short_conv(&prefix)?,
                Some(LayerType::BoundedAttention) => load_bounded_attn(&prefix, li)?,
                _ => load_full_attn(&prefix, Some(li))?,
            };
            // Gemma-2/3 sandwich: `pre_feedforward_layernorm` present →
            // it is the pre-FFN norm, and post_attention/post_feedforward
            // норms apply to the branch OUTPUTS before their residuals.
            let pre_ffn = format!("{prefix}pre_feedforward_layernorm.weight");
            let sandwich = model.tensor(&pre_ffn).is_some();
            layers.push(LayerWeights {
                input_norm: load_f32(model, &format!("{prefix}input_layernorm.weight"), ov)
                    .map_err(err)?,
                post_norm: if sandwich {
                    load_f32(model, &pre_ffn, ov).map_err(err)?
                } else {
                    load_f32(
                        model,
                        &format!("{prefix}post_attention_layernorm.weight"),
                        ov,
                    )
                    .map_err(err)?
                },
                attn_out_norm: if sandwich {
                    Some(
                        load_f32(
                            model,
                            &format!("{prefix}post_attention_layernorm.weight"),
                            ov,
                        )
                        .map_err(err)?,
                    )
                } else {
                    None
                },
                ffn_out_norm: if sandwich {
                    Some(
                        load_f32(
                            model,
                            &format!("{prefix}post_feedforward_layernorm.weight"),
                            ov,
                        )
                        .map_err(err)?,
                    )
                } else {
                    None
                },
                // Gemma-4: learned scalar multiplying the layer output.
                layer_scale: model
                    .tensor(&format!("{prefix}layer_scalar"))
                    .and_then(|_| {
                        load_f32(model, &format!("{prefix}layer_scalar"), ov)
                            .ok()
                            .and_then(|v| v.first().copied())
                    }),
                // FFN always quantized — masks run sparse on quant bytes.
                ffn: build_layer_ffn(model, &arch, li, false, ov)?,
                attn,
            });
        }

        // ── MTP head (optional, spec §2.1) ──
        //
        // The header declaring an MTP head is not the same as the file
        // carrying one. DeepSeek-V4's config announces a next-token predictor
        // whose weights the converter does not map (they are spelled `mtp.N.*`
        // and have none of the canonical projections), so demanding
        // `model.mtp.layers.0.self_attn.q_proj.weight` failed a model that is
        // otherwise complete. Presence in the directory decides.
        let mtp_present = model
            .tensor("model.mtp.layers.0.self_attn.q_proj.weight")
            .is_some()
            || model.tensor("model.mtp.eh_proj.weight").is_some();
        // DeepSeek-V4 writes its own stack under `model.mtp.N.*` — three full
        // layers, not a V3-style single block — so it cannot go through the
        // path below and is loaded by the dsv4 arm instead. Saying the file
        // "carries none" was a false negative worth six gigabytes.
        let dsv4_mtp = model.tensor("model.mtp.0.main_proj.weight").is_some();
        if arch.mtp.is_some() && !mtp_present && !dsv4_mtp {
            tracing::info!(
                "header declares an MTP head but the file carries none — \
                 loading without it"
            );
        }
        // MiMo-V2 carries its own three-layer stack (sidecar or main file),
        // loaded by `mimo_mtp::load_for` below — never this single block.
        let mtp = if let Some(cfg) = arch
            .mtp
            .as_ref()
            .filter(|_| mtp_present && arch.arch_name != "mimo_v2")
        {
            if cfg.num_layers != 1 {
                return Err(CmfError::Parse(format!(
                    "MTP with {} blocks not supported yet (only 1)",
                    cfg.num_layers
                )));
            }
            let p = "model.mtp.";
            let attn = load_full_attn("model.mtp.layers.0.", None)?;
            Some(MtpModule {
                enorm: load_f32(model, &format!("{p}enorm.weight"), ov).map_err(err)?,
                hnorm: load_f32(model, &format!("{p}hnorm.weight"), ov).map_err(err)?,
                eh_proj: load_matrix(model, &format!("{p}eh_proj.weight"), false, ov)?,
                layer: LayerWeights {
                    attn_out_norm: None,
                    ffn_out_norm: None,
                    layer_scale: None,
                    input_norm: load_f32(model, &format!("{p}layers.0.input_layernorm.weight"), ov)
                        .map_err(err)?,
                    post_norm: load_f32(
                        model,
                        &format!("{p}layers.0.post_attention_layernorm.weight"),
                        ov,
                    )
                    .map_err(err)?,
                    // Whatever the block actually carries: DeepSeek's MTP
                    // layer is dense, Qwen3.6's is a full MoE (router + 256
                    // experts + shared). Same builder as a backbone layer.
                    ffn: build_ffn_at(model, &arch, &format!("{p}layers.0."), false, ov)?,
                    attn,
                },
                final_norm: load_f32(model, &format!("{p}norm.weight"), ov).map_err(err)?,
                kv: LayerKvCache::new(arch.num_kv_heads, arch.head_dim),
            })
        } else {
            None
        };

        tracing::info!(
            "Pipeline loaded: {} | {}L ({} linear) | {:.2}B params | storage: {} | MTP: {}",
            arch.arch_name,
            arch.num_layers,
            arch.layer_types
                .iter()
                .filter(|t| matches!(t, LayerType::LinearAttention))
                .count(),
            model.total_param_count() as f64 / 1e9,
            if force_f32 {
                "f32 (masked)"
            } else {
                "quantized mmap"
            },
            if mtp.is_some() { "yes" } else { "no" }
        );

        // KV window: the descriptor's max, capped for dev-box safety;
        // CMF_MAX_SEQ overrides the cap (long-context runs).
        // 8192 was the silent quality cliff of the Qwen3.8 bring-up: at
        // the cap the wgpu token graph declines, the host evicts half the
        // KV, and a GDN hybrid's recurrent state goes stale — the model
        // stays fluent and loses its mind (Django internals, a Turkish
        // essay, an em-dash loop; one failure, three costumes). 32768
        // covers every long-form run we actually ship while keeping the
        // graph's device KV mirror affordable beside the weights;
        // CMF_MAX_SEQ still overrides in either direction.
        let cap = std::env::var("CMF_MAX_SEQ")
            .ok()
            .and_then(|v| v.parse::<usize>().ok())
            .unwrap_or(32_768);
        let max_seq_len = arch.max_position_embeddings.min(cap);

        // Looped Transformer: total virtual layers = physical × num_loops.
        let total_layers = arch.num_layers * arch.num_loops;

        let mut pipeline = Pipeline::new(
            tokenizer,
            PipelineWeights {
                embed_tokens,
                layers,
                lm_head,
                final_norm,
            },
            arch.hidden_size,
            arch.intermediate_size,
            arch.num_attention_heads,
            arch.num_kv_heads,
            arch.head_dim,
            total_layers,
            arch.num_layers, // physical layers in weights
            arch.loop_final_norm,
            arch.vocab_size,
            arch.rms_norm_eps,
            arch.rope_theta as f32,
            arch.norm_style,
            max_seq_len,
            sampler_config,
        );
        let rotary = ((arch.head_dim as f32 * arch.partial_rotary_factor) as usize).max(2);
        pipeline.set_rotary(rotary, arch.rope_theta as f32);
        pipeline.attention_heads_per_layer = arch.attention_heads_per_layer.clone();
        if let Some(yarn) = &arch.yarn {
            pipeline.inv_freq = std::sync::Arc::new(crate::attention::yarn_inv_freq(
                rotary,
                arch.rope_theta as f32,
                yarn.factor,
                yarn.original_max_position_embeddings,
                yarn.beta_fast,
                yarn.beta_slow,
            ));
            pipeline.rope_scale = yarn.attention_factor;
        }
        // Gemma-family extras: embedding scale, attention-scale
        // override, and (Gemma-3) sliding-window layers with their own
        // local RoPE base.
        pipeline.embed_multiplier = arch.embed_multiplier;
        pipeline.logit_multiplier = arch.logit_multiplier;
        if let Some(qpas) = arch.query_pre_attn_scalar {
            pipeline.attn_scale = 1.0 / (qpas as f32).sqrt();
        }
        if let (Some(w), Some(p)) = (arch.sliding_window, arch.sliding_window_pattern) {
            pipeline.swa = Some((w, p));
            if let Some(base) = arch.rope_local_base_freq {
                pipeline.inv_freq_local = Some(std::sync::Arc::new(
                    crate::attention::rope_inv_freq(rotary, base as f32),
                ));
            }
        }
        let explicit_sliding: Vec<bool> = arch
            .layer_types
            .iter()
            .map(|t| matches!(t, cortiq_core::LayerType::SlidingAttention))
            .collect();
        if explicit_sliding.iter().any(|&v| v) {
            pipeline.sliding_layers = Some(explicit_sliding);
            if let Some(w) = arch.sliding_window {
                pipeline.swa = Some((w, usize::MAX));
            }
            let local_rotary = ((arch.head_dim as f32
                * arch
                    .local_partial_rotary_factor
                    .unwrap_or(arch.partial_rotary_factor))
                as usize)
                .max(2);
            pipeline.rotary_dim_local = Some(local_rotary);
            if let Some(base) = arch.rope_local_base_freq {
                pipeline.inv_freq_local = Some(std::sync::Arc::new(
                    crate::attention::rope_inv_freq(local_rotary, base as f32),
                ));
            }
        }
        // Gemma-4: global layers run their own geometry (MQA at
        // global_head_dim) with a proportional RoPE — the first
        // factor·head_dim dims rotate, the zero-padded tail is identity.
        if let (Some(ghd), Some(gkv)) = (arch.global_head_dim, arch.num_global_kv_heads) {
            pipeline.global_attn = Some((ghd, gkv));
            let prf = arch.global_partial_rotary_factor.unwrap_or(1.0);
            let half = ghd / 2;
            let ra = (((prf * ghd as f32) as usize) / 2).min(half);
            let mut f = vec![0.0f32; half];
            for (i, slot) in f.iter_mut().enumerate().take(ra) {
                *slot = 1.0 / (arch.rope_theta as f32).powf(2.0 * i as f32 / ghd as f32);
            }
            pipeline.inv_freq_global = Some(std::sync::Arc::new(f));
            // Re-shape the global layers' KV storage to their geometry.
            // An explicit layer_types map wins over the numeric pattern
            // (explicit tags set swa's pattern to usize::MAX, which
            // would otherwise leave every global cache mis-shaped).
            let global_at = |li: usize| -> bool {
                match &pipeline.sliding_layers {
                    Some(map) => !map.get(li).copied().unwrap_or(false),
                    None => pipeline
                        .swa
                        .map(|(_, p)| p > 0 && p != usize::MAX && (li + 1) % p == 0)
                        .unwrap_or(false),
                }
            };
            for li in 0..arch.num_layers {
                if global_at(li) {
                    pipeline.kv_cache.layers[li] = crate::kv_cache::LayerKvCache::new(gkv, ghd);
                }
            }
        }
        // MLA (DeepSeek-V2): the expand-to-MHA cache holds nh heads of
        // rope+nope dims; rotary covers the rope prefix.
        if let Some(mla) = arch.mla.as_ref() {
            let hd = mla.qk_rope_head_dim + mla.qk_nope_head_dim;
            pipeline.head_dim = hd;
            pipeline.num_kv_heads = arch.num_attention_heads;
            pipeline.rotary_dim = mla.qk_rope_head_dim;
            let half = mla.qk_rope_head_dim / 2;
            let mut f = vec![0.0f32; half];
            for (i, slot) in f.iter_mut().enumerate() {
                *slot = 1.0
                    / (arch.rope_theta as f32).powf(2.0 * i as f32 / mla.qk_rope_head_dim as f32);
            }
            pipeline.inv_freq = std::sync::Arc::new(f);
            for li in 0..arch.num_layers {
                pipeline.kv_cache.layers[li] =
                    crate::kv_cache::LayerKvCache::new(arch.num_attention_heads, hd);
            }
        }
        // Per-layer KV geometry (MiMo-V2): KV heads per layer and V heads
        // narrower than Q/K. The per-layer shapes were checked against the
        // header in load_full_attn; this installs the geometry and gives
        // every layer whose KV head count differs its own cache shape.
        if !owns_its_layers {
            pipeline
                .set_attn_geometry(arch.kv_heads_per_layer.clone(), arch.v_head_dim)
                .map_err(|e| CmfError::Parse(format!("attention geometry: {e}")))?;
        } else if arch.kv_heads_per_layer.is_some() || arch.v_head_dim.is_some() {
            return Err(CmfError::Parse(format!(
                "{}: kv_heads_per_layer / v_head_dim are not supported by its own layer stack",
                arch.arch_name
            )));
        }
        // Learned attention sinks (gpt-oss / MiMo-V2 SWA layers), by tensor
        // presence: `model.layers.N.self_attn.sinks`, one f32 logit per Q
        // head (`attention_sink_bias` is the HF spelling, accepted too).
        // They fold into the grouped softmax; every attention path that
        // cannot carry them (chunk GEMM attend, O(1), the GPU graphs)
        // refuses the layer.
        if !owns_its_layers {
            for li in 0..arch.num_layers {
                let name = [
                    format!("model.layers.{li}.self_attn.sinks"),
                    format!("model.layers.{li}.self_attn.attention_sink_bias"),
                ]
                .into_iter()
                .find(|n| model.tensor(n).is_some());
                if let Some(name) = name {
                    let sinks = load_f32(model, &name, ov).map_err(err)?;
                    pipeline
                        .set_layer_sinks(li, sinks)
                        .map_err(|e| CmfError::Parse(format!("{name}: {e}")))?;
                }
            }
        }
        if pipeline.kv_heads_per_layer.is_some()
            || pipeline.v_head_dim.is_some()
            || pipeline.kv_cache.layers.iter().any(|l| l.sinks.is_some())
        {
            let per_token: usize = pipeline
                .kv_cache
                .layers
                .iter()
                .map(|l| 2 * l.num_kv_heads * l.head_dim * std::mem::size_of::<f32>())
                .sum();
            tracing::info!(
                "attention geometry: kv heads per layer {:?}, v_head_dim {:?}, {} sink layer(s); \
                 f32 KV {} B/token (V padded to head_dim), cap {} tokens",
                pipeline.kv_heads_per_layer,
                pipeline.v_head_dim,
                pipeline
                    .kv_cache
                    .layers
                    .iter()
                    .filter(|l| l.sinks.is_some())
                    .count(),
                per_token,
                pipeline.kv_cache.max_seq_len
            );
        }
        // Per-frequency rope divisors (MiniCPM3 longrope short_factor):
        // served at the native window with the trained per-dim factors.
        // Applied after every inv_freq build (plain, YaRN, MLA).
        if let Some(fac) = &arch.rope_freq_factors {
            let mut f = pipeline.inv_freq.as_ref().clone();
            for (i, v) in f.iter_mut().enumerate() {
                if let Some(&d) = fac.get(i) {
                    *v /= d as f32;
                }
            }
            pipeline.inv_freq = std::sync::Arc::new(f);
        }
        if let Some(selected) = &phase_delta_layers {
            for &li in selected {
                pipeline.kv_cache.layers[li].set_linear_wire_allowed(false);
            }
        }
        pipeline.attn_v_norm = arch.attn_v_norm;
        pipeline.qk_norm_after_rope = arch.qk_norm_after_rope;
        pipeline.final_softcap = arch.final_logit_softcapping.map(|c| c as f32);
        // Cortiq Embryo hierarchical head: cluster matrix → two-level log-probs.
        if let Some(ncl) = arch.head_clusters {
            let cm = load_f32(model, "lm_head.clusters.weight", ov).map_err(err)?;
            if cm.len() != ncl * arch.hidden_size {
                return Err(CmfError::Parse(format!(
                    "lm_head.clusters.weight: {} != {ncl}×{}",
                    cm.len(),
                    arch.hidden_size
                )));
            }
            pipeline.head_clusters = Some(std::sync::Arc::new(cm));
        }
        pipeline.attn_softcap = arch.attn_logit_softcapping.unwrap_or(0.0) as f32;
        pipeline.vmf_cfg = vmf_cfg;
        pipeline.gdn_cfg = gdn_cfg;
        pipeline.kda_cfg = kda_cfg;
        if arch.qwen4_exp.is_some() {
            let (globals, layers, cfg, state) = crate::qwen4_exp::load(model, &arch)?;
            pipeline.qwen4_exp = Some(Box::new((globals, layers, cfg, state)));
        }
        if let Some(gc) = arch.g3n.as_ref() {
            use crate::g3n::{G3nAltUp, G3nGlobals, G3nLaurel, G3nLayer};
            anyhow_like(gc.altup_num_inputs == crate::g3n::ALTUP_N).map_err(|_| {
                CmfError::Parse(format!(
                    "g3n: altup_num_inputs {} != supported {}",
                    gc.altup_num_inputs,
                    crate::g3n::ALTUP_N
                ))
            })?;
            let t = |name: &str| load_matrix(model, name, force_f32, ov);
            let f = |name: &str| load_f32(model, name, ov).map_err(err);
            let mut altup_proj = Vec::new();
            let mut altup_unembed = Vec::new();
            for i in 0..crate::g3n::ALTUP_N - 1 {
                altup_proj.push(t(&format!("model.altup_projections.{i}.weight"))?);
                altup_unembed.push(t(&format!("model.altup_unembed_projections.{i}.weight"))?);
            }
            let first_shared = arch.num_layers.saturating_sub(gc.num_kv_shared_layers);
            let sliding_of = |li: usize| {
                matches!(
                    arch.layer_types.get(li),
                    Some(cortiq_core::LayerType::SlidingAttention)
                )
            };
            let mut g3n_layers = Vec::with_capacity(arch.num_layers);
            for li in 0..arch.num_layers {
                let pfx = format!("model.layers.{li}.");
                let shared = li >= first_shared && first_shared > 0;
                let share_src = if shared {
                    let want = sliding_of(li);
                    (0..first_shared).rev().find(|&j| sliding_of(j) == want)
                } else {
                    None
                };
                g3n_layers.push(G3nLayer {
                    altup: G3nAltUp {
                        router_norm: f(&format!("{pfx}altup.router_norm.weight"))?,
                        modality_router: t(&format!("{pfx}altup.modality_router.weight"))?,
                        prediction_coefs: t(&format!("{pfx}altup.prediction_coefs.weight"))?,
                        correction_coefs: t(&format!("{pfx}altup.correction_coefs.weight"))?,
                        correct_output_scale: f(&format!("{pfx}altup.correct_output_scale"))?,
                    },
                    laurel: G3nLaurel {
                        left: t(&format!("{pfx}laurel.linear_left.weight"))?,
                        right: t(&format!("{pfx}laurel.linear_right.weight"))?,
                        post_norm: f(&format!("{pfx}laurel.post_laurel_norm.weight"))?,
                    },
                    input_norm: f(&format!("{pfx}input_layernorm.weight"))?,
                    post_attn_norm: f(&format!("{pfx}post_attention_layernorm.weight"))?,
                    pre_ffw_norm: f(&format!("{pfx}pre_feedforward_layernorm.weight"))?,
                    post_ffw_norm: f(&format!("{pfx}post_feedforward_layernorm.weight"))?,
                    wq: t(&format!("{pfx}self_attn.q_proj.weight"))?,
                    wk: if shared {
                        None
                    } else {
                        Some(t(&format!("{pfx}self_attn.k_proj.weight"))?)
                    },
                    wv: if shared {
                        None
                    } else {
                        Some(t(&format!("{pfx}self_attn.v_proj.weight"))?)
                    },
                    wo: t(&format!("{pfx}self_attn.o_proj.weight"))?,
                    q_norm: f(&format!("{pfx}self_attn.q_norm.weight"))?,
                    k_norm: if shared {
                        None
                    } else {
                        Some(f(&format!("{pfx}self_attn.k_norm.weight"))?)
                    },
                    kv_share_src: share_src,
                    sliding: sliding_of(li),
                    gate: t(&format!("{pfx}mlp.gate_proj.weight"))?,
                    up: t(&format!("{pfx}mlp.up_proj.weight"))?,
                    down: t(&format!("{pfx}mlp.down_proj.weight"))?,
                    sparsity: gc.activation_sparsity.get(li).copied().unwrap_or(0.0),
                    ple_gate: t(&format!("{pfx}per_layer_input_gate.weight"))?,
                    ple_proj: t(&format!("{pfx}per_layer_projection.weight"))?,
                    post_ple_norm: f(&format!("{pfx}post_per_layer_input_norm.weight"))?,
                });
            }
            let hd = arch.head_dim;
            let globals = G3nGlobals {
                altup_proj,
                altup_unembed,
                ple_embed: t("model.embed_tokens_per_layer.weight")?,
                ple_model_proj: t("model.per_layer_model_projection.weight")?,
                ple_norm: f("model.per_layer_projection_norm.weight")?,
                ple_vocab: gc.ple_vocab,
                ple_dim: gc.ple_dim,
                num_layers: arch.num_layers,
                hidden: arch.hidden_size,
                rms_eps: arch.rms_norm_eps,
                inv_freq_local: crate::attention::rope_inv_freq(
                    hd,
                    arch.rope_local_base_freq.unwrap_or(10_000.0) as f32,
                ),
                inv_freq_global: crate::attention::rope_inv_freq(hd, arch.rope_theta as f32),
                window: arch.sliding_window.unwrap_or(512),
            };
            pipeline.g3n = Some(Box::new((globals, g3n_layers)));
        }
        // DeepSeek-V4.1: its own stack, selected by the preserved source
        // configuration. The generic layer loop cannot represent shared
        // CED/CSA2 state or native Engram tables, so loading failure is
        // fatal instead of falling back to an unrelated attention layout.
        if arch.arch_name == "deepseek_v41" {
            let source = arch.deepseek_v41.as_ref().ok_or_else(|| {
                CmfError::Parse("deepseek_v41: missing preserved source config".into())
            })?;
            let tc = source.get("text_config").unwrap_or(source);
            let usize_of = |key: &str, fallback: usize| {
                tc.get(key)
                    .and_then(|v| v.as_u64())
                    .map(|v| v as usize)
                    .unwrap_or(fallback)
            };
            let f32_of = |key: &str, fallback: f32| {
                tc.get(key)
                    .and_then(|v| v.as_f64())
                    .map(|v| v as f32)
                    .unwrap_or(fallback)
            };
            let usize_any = |keys: &[&str], fallback: usize| {
                keys.iter()
                    .find_map(|key| tc.get(key).and_then(|v| v.as_u64()))
                    .map(|v| v as usize)
                    .unwrap_or(fallback)
            };
            let f32_any = |keys: &[&str], fallback: f32| {
                keys.iter()
                    .find_map(|key| tc.get(key).and_then(|v| v.as_f64()))
                    .map(|v| v as f32)
                    .unwrap_or(fallback)
            };
            let bool_of = |key: &str, fallback: bool| {
                tc.get(key).and_then(|v| v.as_bool()).unwrap_or(fallback)
            };
            let array_of = |key: &str| -> Vec<usize> {
                tc.get(key)
                    .and_then(|v| v.as_array())
                    .map(|a| {
                        a.iter()
                            .filter_map(|v| v.as_u64().map(|x| x as usize))
                            .collect()
                    })
                    .unwrap_or_default()
            };
            let array_alias = |keys: &[&str]| -> Vec<usize> {
                keys.iter()
                    .find_map(|key| {
                        let values = array_of(key);
                        (!values.is_empty()).then_some(values)
                    })
                    .unwrap_or_default()
            };
            let dim = usize_any(&["hidden_size", "dim"], arch.hidden_size);
            let n_layers = usize_any(&["num_hidden_layers", "n_layers"], arch.num_layers);
            let n_heads = usize_any(
                &["num_attention_heads", "n_heads"],
                arch.num_attention_heads,
            );
            let head_dim = usize_any(&["head_dim"], arch.head_dim);
            let rope_head_dim = usize_any(&["rope_head_dim", "qk_rope_head_dim"], 64.min(head_dim));
            let moe_inter = usize_any(
                &[
                    "moe_intermediate_size",
                    "moe_inter_dim",
                    "intermediate_size",
                ],
                arch.intermediate_size,
            );
            let n_experts = usize_any(
                &["n_routed_experts"],
                arch.moe.as_ref().map(|m| m.num_experts).unwrap_or(384),
            );
            let top_k = usize_any(
                &["num_experts_per_tok", "n_activated_experts"],
                arch.moe.as_ref().map(|m| m.top_k).unwrap_or(6),
            );
            let mut ratios = array_of("compress_ratios");
            if ratios.len() >= n_layers {
                ratios.truncate(n_layers);
            } else {
                ratios = (0..n_layers)
                    .map(|li| {
                        if (2..20).contains(&li) {
                            2
                        } else if (20..40).contains(&li) {
                            1
                        } else {
                            0
                        }
                    })
                    .collect();
            }
            let kv_sources = {
                let a = array_alias(&["kv_source_layers", "kv_source_layer_ids"]);
                if a.is_empty() { vec![2, 8, 14, 20] } else { a }
            };
            let index_sources = {
                let a = array_alias(&["index_source_layers", "index_source_layer_ids"]);
                if a.is_empty() {
                    vec![2, 8, 14, 20, 24, 28, 32, 36]
                } else {
                    a
                }
            };
            let engram_layers = array_of("engram_layer_ids");
            let engram_embeddings = array_of("engram_num_embeddings");
            let cfg = crate::dsv41::Dsv41Cfg {
                dim,
                n_heads,
                head_dim,
                rope_head_dim: rope_head_dim.min(head_dim) & !1,
                q_lora_rank: usize_of("q_lora_rank", 1280),
                o_lora_rank: usize_of("o_lora_rank", 1024),
                o_groups: usize_of("o_groups", 8),
                hc_mult: usize_of("hc_mult", 4),
                hc_sinkhorn_iters: usize_of("hc_sinkhorn_iters", 20),
                hc_eps: f32_of("hc_eps", 1e-6),
                norm_eps: f32_any(&["norm_eps", "rms_norm_eps"], arch.rms_norm_eps as f32),
                n_routed_experts: n_experts,
                top_k,
                moe_inter,
                gate_temp: f32_of("gate_temp", 1.0),
                norm_topk_prob: bool_of("norm_topk_prob", true),
                route_scale: f32_any(
                    &["routed_scaling_factor", "route_scale"],
                    arch.moe
                        .as_ref()
                        .and_then(|m| m.routed_scaling_factor)
                        .unwrap_or(1.5),
                ),
                swiglu_limit: f32_of("swiglu_limit", 10.0),
                window: usize_any(
                    &["window_size", "sliding_window"],
                    arch.sliding_window.unwrap_or(128),
                ),
                rope_theta: f32_any(&["rope_theta"], arch.rope_theta as f32),
                compress_rope_theta: f32_any(&["compress_rope_theta"], 160_000.0),
                rope_factor: f32_any(
                    &["rope_factor"],
                    tc.get("rope_scaling")
                        .and_then(|v| v.get("factor"))
                        .and_then(|v| v.as_f64())
                        .map(|v| v as f32)
                        .unwrap_or(16.0),
                ),
                original_seq_len: usize_any(
                    &["original_seq_len", "original_max_position_embeddings"],
                    tc.get("rope_scaling")
                        .and_then(|v| v.get("original_max_position_embeddings"))
                        .and_then(|v| v.as_u64())
                        .map(|v| v as usize)
                        .unwrap_or(65_536),
                ),
                beta_fast: f32_any(
                    &["beta_fast"],
                    tc.get("rope_scaling")
                        .and_then(|v| v.get("beta_fast"))
                        .and_then(|v| v.as_f64())
                        .map(|v| v as f32)
                        .unwrap_or(32.0),
                ),
                beta_slow: f32_any(
                    &["beta_slow"],
                    tc.get("rope_scaling")
                        .and_then(|v| v.get("beta_slow"))
                        .and_then(|v| v.as_f64())
                        .map(|v| v as f32)
                        .unwrap_or(1.0),
                ),
                index_heads: usize_any(&["index_n_heads", "indexer_n_heads"], 32),
                index_head_dim: usize_any(&["index_head_dim", "indexer_head_dim"], 128),
                index_topk: usize_any(&["index_topk", "indexer_topk"], 512),
                candidate_source: usize_any(
                    &["candidate_source_layer", "candidate_source_layer_id"],
                    20,
                ),
                candidate_topk_blocks: usize_any(&["candidate_topk_blocks"], 2048),
                candidate_block_size: usize_any(&["candidate_block_size"], 8),
                kv_sources,
                index_sources,
                compress_ratios: ratios,
                engram_layers,
                engram_vocab: usize_any(&["engram_vocab_size"], 16_000_000),
                engram_embeddings,
                engram_max_ngram: usize_any(&["engram_max_ngram_size"], 4),
                engram_heads: usize_any(&["engram_n_heads"], 8),
                engram_head_dim: usize_any(&["engram_head_dim"], 256),
                engram_compressed_vocab: usize_any(&["engram_compressed_vocab_size"], 99_092),
                engram_pad_id: usize_any(&["engram_pad_id", "engram_pad_token_id"], 2),
                vocab: usize_any(&["vocab_size"], arch.vocab_size),
            };
            let token_map = crate::dsv41::token_map_from_model(model, cfg.vocab);
            let (g, dl, hash) = crate::dsv41::load(model, &cfg, n_layers, token_map)
                .map_err(|e| CmfError::Parse(format!("deepseek_v41: {e}")))?;
            let st = crate::dsv41::Dsv41State::new(&cfg, hash);
            // Vision tensors are optional in text-only exports, but when
            // present they stay mmap-backed through the dedicated tower.
            // Do not make a text-only CMF fail merely because its source
            // config still carries the multimodal section.
            if let Ok(vision_cfg) = crate::dsv41_vision::VisionConfig::from_source(source) {
                if vision_cfg.vision_enabled()
                    && model.tensor("vision.patch_embed.proj.weight").is_some()
                {
                    pipeline.dsv41_vision = Some(
                        crate::dsv41_vision::VisionModel::from_model(model, vision_cfg)
                            .map_err(|e| CmfError::Parse(format!("deepseek_v41 vision: {e}")))?,
                    );
                }
            }
            tracing::info!(
                "deepseek_v41: loaded {} layers, {} KV sources, {} index sources, {} Engram layers; experts remain mmap-backed",
                dl.len(),
                cfg.kv_sources.len(),
                cfg.index_sources.len(),
                cfg.engram_layers.len()
            );
            pipeline.dsv41 = Some(Box::new((g, dl, cfg, st)));
        }
        // DeepSeek-V4: its own stack, selected by the arch name the
        // converter wrote. Loading failure is fatal rather than a silent
        // fallback — the generic loop cannot represent this model at all,
        // so a fallback would decode noise.
        if arch.arch_name == "deepseek_v4" {
            let moe = arch
                .moe
                .as_ref()
                .ok_or_else(|| CmfError::Parse("deepseek_v4: no moe config".into()))?;
            let cfg = crate::dsv4::Dsv4Cfg {
                dim: arch.hidden_size,
                n_heads: arch.num_attention_heads,
                head_dim: arch.head_dim,
                // The rope tail: `partial_rotary_factor` carries it when the
                // conversion recorded it (rd/head_dim), which the tensors
                // cannot reveal. Files converted before that carry 1.0,
                // meaning "unset" here rather than "rotate everything" —
                // for those the release's 64 stands in, which is what they
                // were converted from.
                rope_head_dim: if arch.partial_rotary_factor < 1.0 {
                    (((arch.head_dim as f32 * arch.partial_rotary_factor) as usize) & !1)
                        .clamp(2, arch.head_dim)
                } else {
                    64.min(arch.head_dim)
                },
                // The LoRA ranks and the group count ARE visible in the
                // weights, and reading them there means a re-tuned
                // checkpoint loads without touching this code.
                q_lora_rank: 0,
                o_lora_rank: 0,
                // Derived below from wo_a's shape — the attention output is
                // n_heads*head_dim wide and wo_a takes one group of it per
                // row block, so groups = width / wo_a.cols(). A pinned 8 is
                // right for the release and wrong for anything else, which
                // is exactly what made a toy checkpoint impossible to
                // compare against the reference.
                o_groups: 8,
                hc_mult: 4,
                hc_sinkhorn_iters: 20,
                hc_eps: 1e-6,
                norm_eps: arch.rms_norm_eps as f32,
                n_routed_experts: moe.num_experts,
                top_k: moe.top_k,
                moe_inter: moe.moe_intermediate_size,
                route_scale: moe.routed_scaling_factor.unwrap_or(1.0),
                // config.json's `swiglu_limit`, which the header has no
                // field for. The release ships 10.0; a checkpoint that
                // retunes it would need this read from the config, so it
                // sits next to the other pinned constants rather than
                // hiding inside the expert.
                swiglu_limit: 10.0,
                window: arch.sliding_window.unwrap_or(128),
                index_topk: 512,
                vocab: arch.vocab_size,
            };
            let (g, dl) = crate::dsv4::load(model, &cfg, arch.num_layers)
                .map_err(|e| CmfError::Parse(format!("deepseek_v4: {e}")))?;
            // Read the ranks off the weights that define them: wq_a's
            // rows ARE q_lora_rank, and wo_b's columns are groups x
            // o_lora_rank. A header field could disagree with the file;
            // these cannot.
            let mut cfg = cfg;
            if let Some(l0) = dl.first() {
                cfg.q_lora_rank = l0.wq_a.rows();
                let attn_width = arch.num_attention_heads * arch.head_dim;
                if l0.wo_a.cols() > 0 && attn_width % l0.wo_a.cols() == 0 {
                    cfg.o_groups = (attn_width / l0.wo_a.cols()).max(1);
                }
                cfg.o_lora_rank = l0.wo_b.cols() / cfg.o_groups.max(1);
                cfg.hc_mult = (l0.hc_attn_fn.len() / l0.hc_attn_base.len().max(1)) / cfg.dim.max(1);
                if cfg.hc_mult == 0 {
                    cfg.hc_mult = 4;
                }
            }
            // RoPE rides only the last `rope_head_dim` of each head, and the
            // reference builds its frequencies over THAT width — not over
            // head_dim, which is 512 here. The generic path above sized them
            // by head_dim, giving 1/base^(2i/512) where 1/base^(2i/64) is
            // wanted: every position rotated by the wrong angle.
            //
            // YaRN is applied unconditionally by the reference (its guard is
            // `original_seq_len > 0`, not the sequence length), so it belongs
            // in these frequencies too. Older configs spell the key `type`
            // rather than `rope_type`; when the header carries no profile the
            // release's own numbers stand in, which is better than silently
            // decoding with unscaled frequencies.
            let (yf, yo, ybf, ybs) = match &arch.yarn {
                Some(y) => (
                    y.factor,
                    y.original_max_position_embeddings,
                    y.beta_fast,
                    y.beta_slow,
                ),
                None => {
                    tracing::warn!(
                        "deepseek_v4: the header carries no YaRN profile — \
                         falling back to the release's (factor 16, original \
                         65536, beta 32/1). Re-converting with a build that \
                         reads rope_scaling.type would make this exact."
                    );
                    (16.0, 65536, 32.0, 1.0)
                }
            };
            pipeline.inv_freq = std::sync::Arc::new(crate::attention::yarn_inv_freq(
                cfg.rope_head_dim,
                arch.rope_theta as f32,
                yf,
                yo,
                ybf,
                ybs,
            ));
            // Keep the working set resident. Everything but the routed
            // experts is touched by every token, and of the experts only the
            // ones the task actually routes to — the page cache cannot know
            // that and evicts by age instead.
            if let Ok(stats) = std::env::var("CMF_MOE_PIN") {
                let cover = std::env::var("CMF_MOE_PIN_COVER")
                    .ok()
                    .and_then(|v| v.parse::<f64>().ok())
                    .filter(|&c| c > 0.0 && c <= 1.0)
                    .unwrap_or(0.95);
                let hot = crate::pin::hot_experts(&stats, cover);
                let mut names: Vec<String> = Vec::new();
                for e in &model.tensors {
                    let is_expert = e.name.contains(".mlp.experts.");
                    if !is_expert {
                        names.push(e.name.clone()); // skeleton: always hot
                    }
                }
                let mut kept_experts = 0usize;
                if let Some(hot) = &hot {
                    for (li, experts) in hot {
                        for e in experts {
                            for w in ["gate_proj", "up_proj", "down_proj"] {
                                names.push(format!("model.layers.{li}.mlp.experts.{e}.{w}.weight"));
                            }
                            kept_experts += 1;
                        }
                    }
                }
                let r = crate::pin::pin_tensors(model, &names);
                tracing::info!(
                    "закреплено {:.1} ГБ ({} тензоров, горячих экспертов {kept_experts},                      покрытие {cover}); лимит {}",
                    r.bytes as f64 / 1e9,
                    r.tensors,
                    r.limit
                        .map(|l| format!("{:.1} ГБ", l as f64 / 1e9))
                        .unwrap_or_else(|| "неизвестен".into())
                );
                if r.skipped > 0 {
                    tracing::warn!("не закреплено тензоров: {}", r.skipped);
                }
            }
            let st = crate::dsv4::Dsv4State::new(arch.num_layers);
            // The speculation stack, if the file carries one. Reading it is
            // metadata only — the expert weights stay in the mapping until a
            // draft actually runs — so it costs nothing to know it is there.
            let depth = std::env::var("CMF_DSV4_MTP_DEPTH")
                .ok()
                .and_then(|v| v.parse::<usize>().ok())
                .unwrap_or(3);
            pipeline.dsv4_mtp = crate::dsv4::load_mtp(model, &cfg, depth);
            // Before any trunk pack is built: leave the draft its VRAM.
            crate::dsv4::dspark_reserve_note(&pipeline.dsv4_mtp, &cfg, &dl);
            pipeline.dsv4 = Some(Box::new((g, dl, cfg, st)));
        }
        pipeline.short_conv_cfg = short_conv_cfg;
        pipeline.mtp = mtp;
        if arch.arch_name == "mimo_v2" {
            pipeline.mimo_mtp = crate::pipeline::mimo_mtp::load_for(model, &arch)?;
        }
        pipeline.install_dynamic_routing(model, false);
        // Record the load-time overlay so a later set_active_skill(None)
        // correctly reverts it (the union-diff assumes dyn_active mirrors
        // the live overlay). Blend loads have no single index to revert.
        match ov {
            Overlay::One(sid) => {
                pipeline.dyn_active = model.header.skills.iter().position(|s| &s.id == sid);
            }
            Overlay::Blend(_) => pipeline.dyn_blend_loaded = true,
            Overlay::None => {}
        }
        // B1: apply the measured confidence-calibration temperature, if the
        // file carries one (softmax(logits / T) for reported confidence).
        if let Some(c) = &model.header.calibration {
            pipeline.set_calib_temp(c.temperature);
        }
        // Versioned state wire: every layer carries the operator identity
        // so a peer holding another operator is refused, not guessed at.
        let identity = arch
            .linear_core_identity()
            .and_then(|v| serde_json::to_vec(&v).ok())
            .map(|b| cortiq_core::hash64(&b))
            .unwrap_or(0);
        pipeline.install_wire_identity(identity);
        // Natively bounded anchor: the file's operator, installed from the
        // header (rings sized once, not per prompt). The file governs —
        // a post-hoc O(1) knob on such a file is an error, not a warning.
        if let Some(ac) = &arch.anchor_core {
            pipeline.install_bounded(ac).map_err(CmfError::Parse)?;
            let why = pipeline.o1_refusal().unwrap_or_default();
            let mut knobs: Vec<String> = std::env::vars()
                .filter(|(k, v)| {
                    (k == "CMF_O1" && !(v == "off" || v == "0")) || k.starts_with("CMF_O1_")
                })
                .map(|(k, _)| k)
                .collect();
            knobs.sort();
            if !knobs.is_empty() {
                return Err(CmfError::Parse(format!("{why} (set: {})", knobs.join(", "))));
            }
            if model
                .header
                .provenance
                .as_ref()
                .and_then(|p| p.get("o1_attn"))
                .is_some()
            {
                return Err(CmfError::Parse(format!(
                    "{why} (the header carries a provenance.o1_attn hint)"
                )));
            }
            return Ok(pipeline);
        }
        // O(1) Nyström attention (runtime-level, no format change):
        // env CMF_O1 decides; unset falls through to the converter hint
        // in header.provenance.o1_attn (`cortiq convert --o1`), and
        // CMF_O1=off force-disables even the hint. CLI flags override
        // later via set_o1().
        // A genome's attention operator is its arch (hashed into the
        // trunk); a provenance hint is not part of the genome and must not
        // switch the backbone to Nyström (NF-6 — core refuses the hint on
        // a GENOME file at open; this keeps the loader fail-closed too).
        let genome_hint = model.header.genome.is_some()
            && model
                .header
                .provenance
                .as_ref()
                .and_then(|p| p.get("o1_attn"))
                .is_some();
        if genome_hint {
            return Err(CmfError::Parse(
                "genome file with a provenance.o1_attn hint: the trunk's attention operator is \
                 set by the genome's arch only — refusing"
                    .into(),
            ));
        }
        let o1 = match crate::nystrom::o1_from_env() {
            crate::nystrom::O1Env::Off => None,
            crate::nystrom::O1Env::On(cfg) => Some(cfg),
            crate::nystrom::O1Env::Unset => model
                .header
                .provenance
                .as_ref()
                .and_then(|p| p.get("o1_attn"))
                .and_then(crate::nystrom::O1Cfg::from_json),
        };
        if o1.is_some() {
            if pipeline.attn_softcap > 0.0 {
                return Err(CmfError::Parse(
                    "--o1 with attention-logit soft-capping (Gemma-2) is not supported: \
                     the streaming operator has no capped-score form"
                        .into(),
                ));
            }
            pipeline.set_o1(o1);
        }
        Ok(pipeline)
    }

    /// Record per-skill dynamic-routing metadata: which FFN layers each
    /// skill actually replaces (derived from the tensors present, not
    /// the meta `layers` field), and whether the skill is eligible for
    /// cheap dynamic switching (FFN-only). Called once at load.
    pub(crate) fn install_dynamic_routing(&mut self, model: &Arc<CmfModel>, force_f32: bool) {
        self.model = Some(model.clone());
        self.dyn_force_f32 = force_f32;
        let mut per_skill = Vec::with_capacity(model.header.skills.len());
        for sk in &model.header.skills {
            // A lookup record replaces nothing: its tensors are the key →
            // card table (u64/u32/u8), never weights. No lane, no switch.
            if sk.kind.as_deref() == Some(cortiq_core::knowledge::skill_kind::LOOKUP) {
                per_skill.push(None);
                continue;
            }
            let mut ffn_layers = std::collections::BTreeSet::new();
            let mut non_ffn = false;
            let prefix = format!("skill.{}.", sk.id);
            for t in model.skill_tensors(&sk.id) {
                let rel = &t.name[prefix.len()..]; // e.g. model.layers.20.mlp.down_proj.weight
                let toks: Vec<&str> = rel.split('.').collect();
                if toks.len() >= 5 && toks[0] == "model" && toks[1] == "layers" && toks[3] == "mlp"
                {
                    if let Ok(li) = toks[2].parse::<usize>() {
                        ffn_layers.insert(li);
                        continue;
                    }
                }
                non_ffn = true; // replaces attention / embed / lm_head
            }
            if non_ffn {
                tracing::warn!(
                    "skill '{}' replaces non-FFN tensors — excluded from dynamic \
                     routing (static overlay still works)",
                    sk.id
                );
                per_skill.push(None);
            } else {
                per_skill.push(Some(ffn_layers.into_iter().collect::<Vec<_>>()));
            }
        }
        self.dyn_skill_layers = per_skill;
    }

    /// Switch the overlaid skill for subsequent forwards (dynamic
    /// routing). `idx` = index into model.header.skills; None = backbone.
    /// Rebuilds the FFN of the union of the old and new skill's touched
    /// layers with the new overlay — tensor-source indirection made
    /// dynamic. Cheap: Mapped tensors are re-resolved mmap pointers.
    /// Result is bit-identical to loading the pipeline with that skill.
    ///
    /// Asking for the skill that is already active is a true no-op: the
    /// weights do not change, so the sequence state (KV, rings, recurrent
    /// state, the prefix-reuse key) stays — clearing it here used to wipe
    /// a live conversation on every idempotent call. A real change
    /// rebuilds the touched FFNs first (all or nothing: on an error the
    /// old overlay and state are untouched), then invalidates every state
    /// the old weights produced, including `kv_prefix`.
    pub fn set_active_skill(&mut self, idx: Option<usize>) -> Result<(), CmfError> {
        if self.dyn_active == idx {
            return Ok(());
        }
        let model = self.model.clone().ok_or_else(|| {
            CmfError::Parse("dynamic routing needs a model-backed pipeline".into())
        })?;
        let mut union: std::collections::BTreeSet<usize> = std::collections::BTreeSet::new();
        if let Some(old) = self.dyn_active {
            if let Some(Some(ls)) = self.dyn_skill_layers.get(old) {
                union.extend(ls.iter().copied());
            }
        }
        let new_id: Option<String> = match idx {
            Some(n) => match self.dyn_skill_layers.get(n) {
                Some(Some(ls)) => {
                    union.extend(ls.iter().copied());
                    Some(model.header.skills[n].id.clone())
                }
                _ => {
                    return Err(CmfError::Parse(format!(
                        "skill index {n} not dynamic-eligible"
                    )));
                }
            },
            None => None,
        };
        let ov = match &new_id {
            Some(s) => Overlay::One(s),
            None => Overlay::None,
        };
        let arch = model.arch();
        let mut rebuilt = Vec::with_capacity(union.len());
        for li in union {
            rebuilt.push((
                li,
                build_layer_ffn(&model, arch, li, self.dyn_force_f32, &ov)?,
            ));
        }
        for (li, ffn) in rebuilt {
            self.weights.layers[li].ffn = ffn;
        }
        // Overlay swap changed weights → every cached state is stale.
        self.invalidate_for_weight_change();
        self.dyn_active = idx;
        Ok(())
    }
}

/// Fixed per-sequence state a file declares, from its header alone (no
/// weights loaded): bytes of bounded-anchor rings, bytes of recurrent
/// vectors (S + conv rings), and how many layers still hold a growing
/// per-position KV. A strictly-O(1) file has `growing_layers == 0`.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct PerSequenceState {
    pub bounded_bytes: usize,
    pub recurrent_bytes: usize,
    pub growing_layers: usize,
}

pub fn per_sequence_state_bytes(model: &CmfModel) -> Result<PerSequenceState, String> {
    let arch = model.arch();
    let f = std::mem::size_of::<f32>();
    let mut st = PerSequenceState::default();
    let need = |v: Option<usize>, name: &str| {
        v.ok_or_else(|| format!("linear core needs arch.{name}"))
    };
    for li in 0..arch.num_layers {
        match arch.layer_types.get(li) {
            Some(LayerType::BoundedAttention) => {
                let ac = arch
                    .anchor_core
                    .as_ref()
                    .ok_or("BoundedAttention layer without anchor_core")?;
                st.bounded_bytes += ac.ring_state_elems(arch.num_kv_heads, arch.head_dim) * f;
            }
            Some(LayerType::LinearAttention) => {
                let lc = arch
                    .linear_core
                    .as_ref()
                    .ok_or("LinearAttention layer without linear_core")?;
                let elems = match lc.kind.as_str() {
                    "gated_delta_net" => GdnCfg {
                        num_v_heads: lc.num_heads,
                        num_k_heads: need(arch.linear_num_key_heads, "linear_num_key_heads")?,
                        key_head_dim: need(arch.linear_key_head_dim, "linear_key_head_dim")?,
                        value_head_dim: lc.value_head_dim,
                        conv_kernel: need(arch.linear_conv_kernel_dim, "linear_conv_kernel_dim")?,
                        hidden_size: arch.hidden_size,
                        rms_eps: arch.rms_norm_eps,
                        output_gate_sigmoid: false,
                    }
                    .state_len(),
                    _ => {
                        let nphase = need(lc.nphase, "linear_core.nphase")?;
                        let mut n = lc.num_heads * 2 * nphase * lc.value_head_dim;
                        if let Some(c) = model.tensor(&format!("model.layers.{li}.vmf_attn.conv1d.weight")) {
                            if c.shape.len() == 3 && c.shape[2] >= 2 {
                                n += (c.shape[2] - 1) * arch.hidden_size;
                            }
                        }
                        n
                    }
                };
                st.recurrent_bytes += elems * f;
            }
            Some(LayerType::ShortConv) => {
                let k = need(arch.linear_conv_kernel_dim, "linear_conv_kernel_dim")?;
                st.recurrent_bytes += ShortConvCfg {
                    hidden_size: arch.hidden_size,
                    kernel: k,
                }
                .state_len()
                    * f;
            }
            Some(LayerType::Kda) => {
                st.recurrent_bytes += crate::linear_core::KdaCfg {
                    num_heads: need(arch.linear_num_key_heads, "linear_num_key_heads")?,
                    head_k_dim: need(arch.linear_key_head_dim, "linear_key_head_dim")?,
                    head_v_dim: need(arch.linear_value_head_dim, "linear_value_head_dim")?,
                    conv_kernel: need(arch.linear_conv_kernel_dim, "linear_conv_kernel_dim")?,
                    hidden_size: arch.hidden_size,
                    rms_eps: arch.rms_norm_eps,
                }
                .state_len()
                    * f;
            }
            _ => st.growing_layers += 1,
        }
    }
    Ok(st)
}