cortiq-cli 0.8.25

//! Native Rust converter: a Hugging Face checkpoint (config.json +
//! *.safetensors + tokenizer.json) → a `.cmf` container. No Python, numpy, or
//! torch — reads safetensors and quantizes in Rust, then writes with
//! `cortiq_core::CmfModel::write`.
//!
//! Scope: standard dense transformers (qwen2 / qwen3 / llama / mistral-style,
//! RMSNorm + RoPE + SwiGLU, optional attention biases). Tensor handling is
//! arch-agnostic — 1-D tensors are stored f16, 2-D weights are quantized — so
//! it works by tensor presence without a hard-coded tensor set. Mixture-of-experts
//! is supported (router + per-expert matrices), as is GatedDeltaNet linear
//! attention in the Qwen3.5 hub layout (separate in_proj_qkv/z/a/b) and the
//! fused qwen3_next / AgentWorld layout, whose group-interleaved `in_proj_qkvz`
//! / `in_proj_ba` projections are split natively (`split_fused_gdn`).
//!
//! Not in scope: per-skill delta tensors and task masks — this writes backbones.
//! Those come from the DTG-MA path in `converter/`.

use crate::npy;
use cortiq_core::format::{CMF_VERSION, CmfHeader, TensorSpec, TokenizerBundle};
use cortiq_core::quant::{
    Q2TP_CHUNK, Q2TP_LMAX, Q4TP_LMAX, Q4TP_NIB, bf16_to_f32, f16_to_f32, f32_to_f16,
    q4tp_code_stride, q4tp_put_code,
};
use cortiq_core::types::{
    LayerType, LinearCoreConfig, ModelArch, MoeConfig, NormStyle, PrismAffineConfig,
    PrismHadamardConfig, QuantType, Qwen4ExpConfig, TensorDtype, YarnConfig,
};
use std::collections::{HashMap, HashSet};
use std::fs;
use std::io::Read;
use std::path::{Component, Path, PathBuf};
use std::sync::Mutex;
use std::sync::atomic::{AtomicU8, AtomicU32, AtomicUsize, Ordering};
use std::time::Duration;

const GROUP_SIZE: usize = 32;
/// Smallest normal f16 — floor for degenerate (all-zero) rows so the stored
/// scale never underflows to a subnormal the reader would read back as 0.
const F16_TINY: f32 = 6.103_515_6e-5;
/// Local source-shard consumption is deliberately explicit because it removes
/// the original safetensors files after their output checkpoint is durable.
/// The V4.1 pod conversion sets this when the source and output cannot coexist
/// under the disk budget; ordinary conversions preserve their source files.
const CONSUME_SOURCE_SHARDS_ENV: &str = "CMF_CONSUME_SOURCE_SHARDS";
/// Opt-in certificate directory for a local source whose shards are still
/// being downloaded by another process.  The sibling `source-files.json`
/// pins every filename, byte length and SHA256 before a marker can authorize
/// opening that shard.
const LOCAL_READY_DIR_ENV: &str = "CMF_LOCAL_READY_DIR";
/// Test/operator override for the ready-marker poll interval.  Production
/// uses the conservative default so a marker producer is not busy-polled.
const LOCAL_READY_POLL_MS_ENV: &str = "CMF_LOCAL_READY_POLL_MS";
const LOCAL_READY_POLL_MS_DEFAULT: u64 = 2_000;
/// Optional comma-separated source basenames to process first.  This only
/// affects the streamed source path; an unset variable preserves the index's
/// validated stable order.
const SOURCE_SHARD_PRIORITY_ENV: &str = "CMF_SOURCE_SHARD_PRIORITY";

fn consume_source_shards_enabled() -> bool {
    std::env::var(CONSUME_SOURCE_SHARDS_ENV)
        .ok()
        .is_some_and(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes"))
}

#[derive(Clone, Debug, PartialEq, Eq)]
struct SourceFileExpectation {
    size: u64,
    sha256: String,
}

fn local_ready_dir_from_env() -> Option<PathBuf> {
    std::env::var_os(LOCAL_READY_DIR_ENV)
        .map(PathBuf::from)
        .filter(|p| !p.as_os_str().is_empty())
}

fn local_ready_poll_ms() -> u64 {
    std::env::var(LOCAL_READY_POLL_MS_ENV)
        .ok()
        .and_then(|v| v.trim().parse::<u64>().ok())
        .unwrap_or(LOCAL_READY_POLL_MS_DEFAULT)
}

/// The source manifest lives next to the ready directory (`/root/dsv41` in
/// the pod layout).  Keeping this derived from the one opt-in environment
/// variable avoids a second path that could accidentally point at a
/// different certificate set.
fn local_ready_manifest_path(ready_dir: &Path) -> PathBuf {
    ready_dir
        .parent()
        .filter(|p| !p.as_os_str().is_empty())
        .unwrap_or_else(|| Path::new("."))
        .join("source-files.json")
}

fn ensure_source_filename(name: &str) -> anyhow::Result<()> {
    let path = Path::new(name);
    anyhow::ensure!(
        !name.is_empty()
            && !name.contains('\\')
            && path.is_relative()
            && path.components().count() == 1
            && matches!(path.components().next(), Some(Component::Normal(_))),
        "local-ready source filename must be a plain basename, got {name:?}"
    );
    Ok(())
}

fn parse_sha256(value: &str, context: &str) -> anyhow::Result<String> {
    let normalized = value.trim().to_ascii_lowercase();
    anyhow::ensure!(
        normalized.len() == 64 && normalized.bytes().all(|b| b.is_ascii_hexdigit()),
        "{context}: SHA256 must be exactly 64 hexadecimal characters"
    );
    Ok(normalized)
}

/// Read the pinned parent manifest.  The production manifest is a JSON object
/// keyed by shard basename; each value contains `size` and `sha256`.  The
/// required set comes from the local model index, so unrelated files in the
/// manifest do not silently become conversion inputs.
fn read_source_file_manifest(
    path: &Path,
    required_shards: &[String],
) -> anyhow::Result<HashMap<String, SourceFileExpectation>> {
    let text = fs::read_to_string(path)
        .map_err(|e| anyhow::anyhow!("read local-ready source manifest {}: {e}", path.display()))?;
    let value: serde_json::Value = serde_json::from_str(&text).map_err(|e| {
        anyhow::anyhow!("parse local-ready source manifest {}: {e}", path.display())
    })?;
    let object = value.as_object().ok_or_else(|| {
        anyhow::anyhow!(
            "local-ready source manifest {} must be a JSON object keyed by filename",
            path.display()
        )
    })?;
    let mut expected = HashMap::with_capacity(object.len());
    for (name, record) in object {
        ensure_source_filename(name)?;
        let record = record
            .as_object()
            .ok_or_else(|| anyhow::anyhow!("manifest entry {name:?} is not an object"))?;
        let size = record
            .get("size")
            .and_then(serde_json::Value::as_u64)
            .ok_or_else(|| anyhow::anyhow!("manifest entry {name:?} has no u64 size"))?;
        let sha256 = record
            .get("sha256")
            .and_then(serde_json::Value::as_str)
            .ok_or_else(|| anyhow::anyhow!("manifest entry {name:?} has no sha256"))
            .and_then(|v| parse_sha256(v, &format!("manifest entry {name:?}")))?;
        anyhow::ensure!(
            expected
                .insert(name.clone(), SourceFileExpectation { size, sha256 })
                .is_none(),
            "duplicate local-ready source manifest entry {name:?}"
        );
    }
    for name in required_shards {
        ensure_source_filename(name)?;
        anyhow::ensure!(
            expected.contains_key(name),
            "local-ready source manifest {} has no entry for indexed shard {name}",
            path.display()
        );
    }
    Ok(expected)
}

fn ready_marker_path(ready_dir: &Path, filename: &str) -> anyhow::Result<PathBuf> {
    ensure_source_filename(filename)?;
    Ok(ready_dir.join(format!("{filename}.json")))
}

fn read_ready_marker(
    path: &Path,
    filename: &str,
    expected: &SourceFileExpectation,
) -> anyhow::Result<()> {
    let text = fs::read_to_string(path)
        .map_err(|e| anyhow::anyhow!("read local-ready marker {}: {e}", path.display()))?;
    let value: serde_json::Value = serde_json::from_str(&text)
        .map_err(|e| anyhow::anyhow!("parse local-ready marker {}: {e}", path.display()))?;
    let object = value
        .as_object()
        .ok_or_else(|| anyhow::anyhow!("local-ready marker {} is not an object", path.display()))?;
    let marker_name = object
        .get("filename")
        .and_then(serde_json::Value::as_str)
        .ok_or_else(|| anyhow::anyhow!("local-ready marker {} has no filename", path.display()))?;
    anyhow::ensure!(
        marker_name == filename,
        "local-ready marker {} names {marker_name:?}, expected {filename:?}",
        path.display()
    );
    let marker_size = object
        .get("size")
        .and_then(serde_json::Value::as_u64)
        .ok_or_else(|| anyhow::anyhow!("local-ready marker {} has no u64 size", path.display()))?;
    anyhow::ensure!(
        marker_size == expected.size,
        "local-ready marker {} size {} disagrees with pinned size {}",
        path.display(),
        marker_size,
        expected.size
    );
    let marker_sha = object
        .get("sha256")
        .and_then(serde_json::Value::as_str)
        .ok_or_else(|| anyhow::anyhow!("local-ready marker {} has no sha256", path.display()))
        .and_then(|v| parse_sha256(v, &format!("local-ready marker {}", path.display())))?;
    anyhow::ensure!(
        marker_sha == expected.sha256,
        "local-ready marker {} SHA256 disagrees with pinned source manifest",
        path.display()
    );
    Ok(())
}

/// Hash one ready source shard with a fixed-size buffer.  This is intentionally
/// done before mmap: a marker is a certificate boundary, while the second
/// metadata check catches a producer that changed a file after certification.
fn sha256_file(path: &Path) -> anyhow::Result<(u64, String)> {
    use sha2::{Digest, Sha256};
    let mut file = fs::File::open(path)
        .map_err(|e| anyhow::anyhow!("open local-ready source {}: {e}", path.display()))?;
    let before = file.metadata()?;
    let mut hasher = Sha256::new();
    let mut buf = vec![0u8; 8 * 1024 * 1024];
    let mut size = 0u64;
    loop {
        let n = file.read(&mut buf)?;
        if n == 0 {
            break;
        }
        hasher.update(&buf[..n]);
        size = size.checked_add(n as u64).ok_or_else(|| {
            anyhow::anyhow!("local-ready source {} size overflow", path.display())
        })?;
    }
    let after = file.metadata()?;
    anyhow::ensure!(
        before.len() == after.len() && before.modified().ok() == after.modified().ok(),
        "local-ready source {} changed while it was being certified",
        path.display()
    );
    Ok((size, format!("{:x}", hasher.finalize())))
}

/// Validate a marker and its source bytes.  Callers only pass the returned
/// path to `open_safetensors` after this function succeeds, so incomplete
/// `.aria2`/partial files can never enter the mmap path.
fn validate_local_ready_source(
    ready_dir: &Path,
    source_dir: &Path,
    filename: &str,
    expected: &SourceFileExpectation,
) -> anyhow::Result<PathBuf> {
    let marker = ready_marker_path(ready_dir, filename)?;
    read_ready_marker(&marker, filename, expected)?;
    let source = source_dir.join(filename);
    anyhow::ensure!(
        !PathBuf::from(format!("{}.aria2", source.display())).exists(),
        "local-ready source {} still has an aria2 partial marker",
        source.display()
    );
    let metadata = fs::metadata(&source).map_err(|e| {
        anyhow::anyhow!(
            "local-ready marker exists for missing source {}: {e}",
            source.display()
        )
    })?;
    anyhow::ensure!(
        metadata.is_file() && metadata.len() == expected.size,
        "local-ready source {} has {} bytes, expected {}",
        source.display(),
        metadata.len(),
        expected.size
    );
    let (actual_size, actual_sha) = sha256_file(&source)?;
    anyhow::ensure!(
        actual_size == expected.size && actual_sha == expected.sha256,
        "local-ready source {} failed pinned SHA256/size validation",
        source.display()
    );
    Ok(source)
}

fn wait_for_local_ready_source(
    ready_dir: &Path,
    source_dir: &Path,
    filename: &str,
    expected: &SourceFileExpectation,
) -> anyhow::Result<PathBuf> {
    let marker = ready_marker_path(ready_dir, filename)?;
    loop {
        if marker.exists() {
            return validate_local_ready_source(ready_dir, source_dir, filename, expected);
        }
        std::thread::sleep(Duration::from_millis(local_ready_poll_ms()));
    }
}

/// Round a scale to f16 precision (the reader stores/uses it as f16), so the
/// quantized values are computed against the *same* scale the reader dequantizes
/// with. This is what the reference converter does; without it `q` and the
/// stored scale disagree and inference degrades to garbage.
fn f16_scale(raw: f32) -> f32 {
    f16_to_f32(f32_to_f16(raw)).max(F16_TINY)
}

/// tiktoken pre-tokenization pattern (Kimi family, o200k lineage) —
/// verbatim from tokenization_kimi.py; tiktoken itself compiles this
/// with fancy-regex, the same engine the runtime tokenizer uses.
const TIKTOKEN_KIMI_PAT: &str = concat!(
    r"[\p{Han}]+",
    r"|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*",
    r"[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
    r"|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+",
    r"[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
    r"|\p{N}{1,3}",
    r"| ?[^\s\p{L}\p{N}]+[\r\n]*",
    r"|\s*[\r\n]+",
    r"|\s+(?!\S)",
    r"|\s+",
);

/// GPT-2 byte-level alphabet: raw byte → printable char (identity for
/// the printable ranges, 256+n in first-free order for the rest).
fn byte_level_table() -> [char; 256] {
    let mut table = ['\0'; 256];
    let mut n = 0u32;
    for b in 0..=255u32 {
        let printable =
            (0x21..=0x7E).contains(&b) || (0xA1..=0xAC).contains(&b) || (0xAE..=0xFF).contains(&b);
        table[b as usize] = if printable {
            char::from_u32(b).unwrap()
        } else {
            let c = char::from_u32(256 + n).unwrap();
            n += 1;
            c
        };
    }
    table
}

/// Build an HF tokenizer.json from a tiktoken rank table (Kimi family).
///
/// tiktoken stores `base64(token_bytes) rank` per line and no merge
/// list — the merges ARE the ranks. Recovery (transformers' tiktoken
/// converter): BPE-split every multi-byte token using only merges of
/// strictly lower rank; the two parts it stops at are that token's
/// merge rule. Specials come from tokenizer_config.added_tokens_decoder.
fn tiktoken_to_tokenizer_json(model: &str, tok_cfg: &serde_json::Value) -> anyhow::Result<String> {
    use base64::Engine as _;
    use std::collections::HashMap;
    let b64 = base64::engine::general_purpose::STANDARD;
    let mut ranks: HashMap<Vec<u8>, u32> = HashMap::new();
    let mut ordered: Vec<(Vec<u8>, u32)> = Vec::new();
    for line in model.lines() {
        let line = line.trim();
        if line.is_empty() {
            continue;
        }
        let (tok, rank) = line
            .rsplit_once(' ')
            .ok_or_else(|| anyhow::anyhow!("tiktoken: bad line {line:?}"))?;
        let bytes = b64
            .decode(tok)
            .map_err(|e| anyhow::anyhow!("tiktoken: bad base64 {tok:?}: {e}"))?;
        let rank: u32 = rank.parse()?;
        ranks.insert(bytes.clone(), rank);
        ordered.push((bytes, rank));
    }
    anyhow::ensure!(!ordered.is_empty(), "tiktoken: empty rank table");
    ordered.sort_by_key(|(_, r)| *r);

    let table = byte_level_table();
    let to_bl = |bytes: &[u8]| -> String { bytes.iter().map(|&b| table[b as usize]).collect() };

    // Merge recovery: split `token` greedily by lowest-rank pair, using
    // only merges of rank < the token's own.
    let bpe_parts = |token: &[u8], max_rank: u32| -> Vec<Vec<u8>> {
        let mut parts: Vec<Vec<u8>> = token.iter().map(|&b| vec![b]).collect();
        loop {
            let mut best: Option<(usize, u32)> = None;
            for i in 0..parts.len() - 1 {
                let cat = [parts[i].as_slice(), parts[i + 1].as_slice()].concat();
                if let Some(&r) = ranks.get(cat.as_slice()) {
                    if r < max_rank && best.map(|(_, br)| r < br).unwrap_or(true) {
                        best = Some((i, r));
                    }
                }
            }
            match best {
                Some((i, _)) => {
                    let b = parts.remove(i + 1);
                    parts[i].extend(b);
                }
                None => break,
            }
        }
        parts
    };

    let mut vocab = serde_json::Map::new();
    let mut merges: Vec<serde_json::Value> = Vec::new();
    let mut skipped = 0usize;
    for (bytes, rank) in &ordered {
        vocab.insert(to_bl(bytes), serde_json::json!(rank));
        if bytes.len() > 1 {
            let parts = bpe_parts(bytes, *rank);
            if parts.len() == 2 {
                merges.push(serde_json::json!([to_bl(&parts[0]), to_bl(&parts[1])]));
            } else {
                // Unreachable-by-merges token (rare): encodable only as
                // a whole pre-token piece; keep it in the vocab.
                skipped += 1;
            }
        }
    }
    if skipped > 0 {
        tracing::warn!("tiktoken: {skipped} tokens yielded no merge rule (kept vocab-only)");
    }

    let mut added: Vec<serde_json::Value> = Vec::new();
    if let Some(atd) = tok_cfg
        .get("added_tokens_decoder")
        .and_then(|v| v.as_object())
    {
        for (id, tok) in atd {
            if let (Ok(id), Some(content)) = (
                id.parse::<u32>(),
                tok.get("content").and_then(|c| c.as_str()),
            ) {
                added.push(serde_json::json!({
                    "id": id, "content": content, "special": true
                }));
            }
        }
    }

    let json = serde_json::json!({
        "version": "1.0",
        "added_tokens": added,
        "pre_tokenizer": {
            "type": "Split",
            "pattern": { "Regex": TIKTOKEN_KIMI_PAT },
            "behavior": "Isolated",
            "invert": false
        },
        "decoder": { "type": "ByteLevel" },
        "model": {
            "type": "BPE",
            "vocab": serde_json::Value::Object(vocab),
            "merges": merges
        }
    });
    Ok(serde_json::to_string(&json)?)
}

/// Canonicalize a source tensor name to the CMF layout the runtime expects, or
/// `None` to skip it. Multimodal wrappers (Qwen3.5) nest the text model under
/// `model.language_model.*`; vision (`*.visual.*`) and the MTP head (`mtp.*`) are
/// dropped — plain greedy decoding is correct without MTP.
pub(crate) fn canon_name(raw: &str) -> Option<String> {
    if raw.contains(".visual.") || raw.starts_with("visual.") {
        return None;
    }
    // MTP head: kept, and renamed to the layout the loader reads. Qwen3.6
    // spells the projection `fc` and its two input norms
    // `pre_fc_norm_{embedding,hidden}`; the loader (written against the
    // DeepSeek spelling) wants `eh_proj`, `enorm`, `hnorm`. Everything
    // under `layers.0.` passes through — including the MoE mlp, which the
    // block carries here rather than a dense one.
    if let Some(rest) = raw.strip_prefix("mtp.").or_else(|| {
        raw.strip_prefix("model.mtp.")
            .or_else(|| raw.split_once(".mtp.").map(|(_, r)| r))
    }) {
        let mapped = match rest {
            "fc.weight" => "eh_proj.weight".to_string(),
            "pre_fc_norm_embedding.weight" => "enorm.weight".to_string(),
            "pre_fc_norm_hidden.weight" => "hnorm.weight".to_string(),
            other => other.to_string(),
        };
        return Some(format!("model.mtp.{mapped}"));
    }
    // Gemma-4 multimodal towers (text tower converts alone).
    for pfx in [
        "model.vision_embedder.",
        "model.embed_audio.",
        "model.embed_vision.",
        // Kimi-K3 vision tower + projector (text tower converts alone).
        "vision_tower.",
        "mm_projector.",
        // Gemma-3n multimodal towers.
        "model.audio_tower.",
        "model.vision_tower.",
        // MiniCPM-V projector/merger.  This generic text-only path has no
        // image placeholder or canvas-MRoPE runtime, so keeping it would
        // create unreadable dead payloads beside the text tower.
        "model.merger.",
        // Qwen3.5-VL vision tower (Ornith-1.5 among them): 333 tensors
        // that would ride into the file as dead weight under their raw
        // names — the text tower converts alone.
        "model.visual.",
    ] {
        if raw.starts_with(pfx) {
            return None;
        }
    }
    for pfx in [
        "model.language_model.",
        "language_model.model.",
        "language_model.",
    ] {
        if let Some(rest) = raw.strip_prefix(pfx) {
            // Re-enter with the wrapper stripped so every later rewrite
            // (Kimi block_sparse_moe, shared_experts, expert_bias) still
            // applies to nested checkpoints.
            return canon_name(&format!("model.{rest}"));
        }
    }
    // ── DeepSeek-V4/V4.1 (`deepseek_v4*`) ─────────────────────────────
    // Its checkpoint drops the `model.` wrapper entirely and spells the
    // blocks `attn`/`ffn`, so the names arrive at top level: `embed`,
    // `head`, `norm`, `layers.N.attn.*`, `layers.N.ffn.*`. Everything
    // below is a rename into the layout the loader already reads; the
    // architecture-specific tensors (compressor, indexer, hyper-
    // connections) keep their spelling under the layer prefix so the
    // engine can find them once those nodes exist.
    if raw == "embed.weight" {
        return Some("model.embed_tokens.weight".into());
    }
    if raw == "head.weight" {
        return Some("lm_head.weight".into());
    }
    if raw == "norm.weight" {
        return Some("model.norm.weight".into());
    }
    // The hyper-connection head tensors are model-global (hc_head_base /
    // _fn / _scale) — pass them through under `model.`.
    if raw.starts_with("hc_head_") {
        return Some(format!("model.{raw}"));
    }
    if let Some(rest) = raw.strip_prefix("layers.") {
        // `layers.N.` + the rest; N is the layer index.
        if let Some((li, tail)) = rest.split_once('.') {
            if li.chars().all(|c| c.is_ascii_digit()) {
                let mapped = match tail {
                    "attn_norm.weight" => "input_layernorm.weight".to_string(),
                    "ffn_norm.weight" => "post_attention_layernorm.weight".to_string(),
                    // Router: weight + the noaux_tc selection bias, which
                    // the loader reads as `mlp.expert_bias`.
                    "ffn.gate.weight" => "mlp.gate.weight".to_string(),
                    "ffn.gate.bias" => "mlp.expert_bias".to_string(),
                    // V4.1's vision-language router carries a second bias
                    // plane; keep it beside the ordinary expert bias so the
                    // multimodal runtime can apply it explicitly.
                    "ffn.gate.bias_vl" => "mlp.expert_bias_vl".to_string(),
                    // Hash-routed layers carry a token-id → expert table
                    // instead of a learned gate.
                    "ffn.gate.tid2eid" => "mlp.tid2eid".to_string(),
                    other => {
                        // experts.E.w{1,3,2} → experts.E.{gate,up,down}_proj
                        // (DeepSeek's w1 = gate, w3 = up, w2 = down), and
                        // the same for the single shared expert.
                        let mut t = other
                            .replace("ffn.shared_experts.", "mlp.shared_expert.")
                            .replace("ffn.experts.", "mlp.experts.");
                        if t.starts_with("mlp.experts.") || t.starts_with("mlp.shared_expert.") {
                            t = t
                                .replace(".w1.weight", ".gate_proj.weight")
                                .replace(".w3.weight", ".up_proj.weight")
                                .replace(".w2.weight", ".down_proj.weight")
                                .replace(".w1.scale", ".gate_proj.scale")
                                .replace(".w3.scale", ".up_proj.scale")
                                .replace(".w2.scale", ".down_proj.scale");
                        }
                        // Attention keeps DeepSeek's own spelling: the
                        // double-LoRA q/o, the compressed KV, the sink,
                        // the compressor and the sparse indexer have no
                        // equivalent in any arch already supported, so
                        // renaming them would invent a layout nothing
                        // reads. `self_attn.` is the prefix every loader
                        // looks under.
                        t = t.replace("attn.", "self_attn.");
                        t
                    }
                };
                return Some(format!("model.layers.{li}.{mapped}"));
            }
        }
    }

    // Kimi (Kimi Linear / Kimi-K3) MoE block: mixtral-style w1/w3/w2
    // experts and a router with the noaux_tc selection bias.
    // `block_sparse_moe` is Kimi-exclusive among supported archs.
    if raw.contains(".block_sparse_moe.") {
        let mut n = raw
            .replace(
                ".block_sparse_moe.gate.e_score_correction_bias",
                ".mlp.expert_bias",
            )
            .replace(".block_sparse_moe.shared_experts.", ".mlp.shared_expert.")
            .replace(".block_sparse_moe.", ".mlp.");
        if n.contains(".mlp.experts.") || n.contains(".mlp.shared_expert.") {
            n = n
                .replace(".w1.weight", ".gate_proj.weight")
                .replace(".w3.weight", ".up_proj.weight")
                .replace(".w2.weight", ".down_proj.weight");
        }
        return Some(n);
    }
    // Laguna stores the router's auxiliary-loss-free selection bias under
    // `experts`, although it belongs to the router mathematically. CMF keeps
    // the canonical bias beside `mlp.gate.weight`.
    if raw.contains(".mlp.shared_experts.") {
        return Some(raw.replace(".mlp.shared_experts.", ".mlp.shared_expert."));
    }
    if raw.ends_with(".mlp.experts.e_score_correction_bias") {
        return Some(raw.replace(".mlp.experts.e_score_correction_bias", ".mlp.expert_bias"));
    }
    // HunYuan dense (Hy-MT2-1.8B/7B, `hunyuan_v1_dense`): the per-head q/k
    // norms are spelled `query_layernorm` / `key_layernorm`; the loader
    // reads `q_norm` / `k_norm`. The ORDER (norm after RoPE) rides in the
    // header flag `qk_norm_after_rope`, not in the names.
    if raw.contains(".self_attn.query_layernorm.") || raw.contains(".self_attn.key_layernorm.") {
        return Some(
            raw.replace(".self_attn.query_layernorm.", ".self_attn.q_norm.")
                .replace(".self_attn.key_layernorm.", ".self_attn.k_norm."),
        );
    }
    // HunYuan hy_v3 MoE (Hy-MT2-30B-A3B): the router is `mlp.router.gate`,
    // the always-on shared expert `mlp.shared_mlp`; `mlp.expert_bias` (the
    // noaux_tc selection bias) is already the canonical spelling.
    if raw.contains(".mlp.router.gate.") || raw.contains(".mlp.shared_mlp.") {
        return Some(
            raw.replace(".mlp.router.gate.", ".mlp.gate.")
                .replace(".mlp.shared_mlp.", ".mlp.shared_expert."),
        );
    }
    Some(lfm2_canon(raw))
}

/// Prism/Bonsai uses the same text canonical layout as the ordinary Qwen
/// reader, but its checkpoint wraps the language model one level deeper and
/// carries a real vision tower.  Keep the generic `canon_name` multimodal
/// policy unchanged (older readers intentionally drop vision); only this
/// explicit architecture path maps the retained tower into `vis.*`.
fn canon_name_for_arch(arch: &ModelArch, raw: &str, towers: MimoTowers) -> Option<String> {
    // MiMo-V2's rules (router bias, sinks, tower/MTP drops) are gated on the
    // arch so the generic canon policy of every other family is untouched.
    if arch.arch_name == MIMO_V2 {
        return mimo_v2_canon(raw, towers);
    }
    if arch.arch_name == "spark2_5" {
        return spark25_canon(raw);
    }
    if arch.arch_name.eq_ignore_ascii_case("whisper")
        && (raw.starts_with("model.encoder.") || raw.starts_with("model.decoder."))
    {
        return Some(raw.to_string());
    }
    if arch.prism_hadamard.is_none() {
        return canon_name(raw);
    }
    if let Some(rest) = raw.strip_prefix("language_model.") {
        if rest == "lm_head.weight" || rest == "lm_head.bias" {
            return Some(rest.to_string());
        }
        if let Some(rest) = rest.strip_prefix("model.") {
            return canon_name(&format!("model.{rest}"));
        }
    }
    let raw = raw.strip_prefix("model.").unwrap_or(raw);
    if let Some(rest) = raw.strip_prefix("vision_tower.") {
        // The Qwen3.5 tower already follows the runtime's patch_embed /
        // blocks / merger naming; only the source namespace is different.
        if let Some(rest) = rest.strip_prefix("patch_embed.proj.") {
            return Some(format!("vis.patch_embed.{rest}"));
        }
        return Some(format!("vis.{rest}"));
    }
    canon_name(raw)
}

/// Spark-X2.5 (`spark2_5`) onto the canonical layout: the embedding is
/// spelled `model.embedding`, attention fuses Q/K/V as `q_k_v_proj` (split
/// downstream by the `qkv_proj` path) and names its output `out_proj`. The
/// head-wise gate `self_attn.g_proj` is already the canonical name.
fn spark25_canon(raw: &str) -> Option<String> {
    if raw == "model.embedding.weight" {
        return Some("model.embed_tokens.weight".into());
    }
    Some(
        raw.replace(".self_attn.q_k_v_proj.", ".self_attn.qkv_proj.")
            .replace(".self_attn.out_proj.", ".self_attn.o_proj."),
    )
}

/// Map LFM2 / LFM2-MoE vendor tensor names onto CMF's canonical (Qwen2)
/// layout so the standard loader reads them unchanged. Every substring
/// below is LFM2-exclusive among the supported architectures, so the
/// rewrite never touches another model's tensors. Returns the name
/// verbatim for non-LFM2 checkpoints.
///
///   operator_norm → input_layernorm      ffn_norm → post_attention_layernorm
///   embedding_norm → norm                 self_attn.out_proj → self_attn.o_proj
///   self_attn.{q,k}_layernorm → {q,k}_norm
///   conv.{in_proj,conv,out_proj} → short_conv.*
///   feed_forward.gate/expert_bias/experts.N → mlp.*
///   feed_forward.w1/w3/w2 → mlp.{gate,up,down}_proj (dense + per expert)
fn lfm2_canon(name: &str) -> String {
    let is_lfm2 = name == "model.embedding_norm.weight"
        || name.contains(".operator_norm")
        || name.contains(".ffn_norm")
        || name.contains(".feed_forward.")
        || name.contains(".conv.")
        || name.contains(".self_attn.out_proj")
        || name.contains(".self_attn.q_layernorm")
        || name.contains(".self_attn.k_layernorm");
    if !is_lfm2 {
        return name.to_string();
    }
    if name == "model.embedding_norm.weight" {
        return "model.norm.weight".to_string();
    }
    let mut n = name.to_string();
    n = n.replace(".operator_norm.", ".input_layernorm.");
    n = n.replace(".ffn_norm.", ".post_attention_layernorm.");
    n = n.replace(".self_attn.out_proj.", ".self_attn.o_proj.");
    n = n.replace(".self_attn.q_layernorm.", ".self_attn.q_norm.");
    n = n.replace(".self_attn.k_layernorm.", ".self_attn.k_norm.");
    n = n.replace(".conv.in_proj.", ".short_conv.in_proj.");
    n = n.replace(".conv.out_proj.", ".short_conv.out_proj.");
    n = n.replace(".conv.conv.", ".short_conv.conv.");
    // FFN: router/bias/experts first, then the dense fallback, then the
    // w1/w3/w2 → gate/up/down rename (applies to both mlp.wK and
    // mlp.experts.N.wK). Order matters — the experts substring carries
    // `.feed_forward.` so it must run before the bare `.feed_forward.`.
    n = n.replace(".feed_forward.gate.weight", ".mlp.gate.weight");
    n = n.replace(".feed_forward.expert_bias", ".mlp.expert_bias");
    n = n.replace(".feed_forward.experts.", ".mlp.experts.");
    n = n.replace(".feed_forward.", ".mlp.");
    n = n.replace(".w1.weight", ".gate_proj.weight");
    n = n.replace(".w3.weight", ".up_proj.weight");
    n = n.replace(".w2.weight", ".down_proj.weight");
    n
}

/// Small, noise-sensitive 2-D projections the reference converter keeps at f16
/// (a bit-flip there is costly): the GDN a/b gate projections and MoE routers.
/// Tensors that must not be quantized OR narrowed: lookup tables whose
/// values are indices, not magnitudes. f16 is exact only to 2048, and
/// DeepSeek-V4's table holds expert ids per vocabulary id (129 280 rows).
fn force_f32(name: &str) -> bool {
    name.ends_with(".tid2eid")
        // MiMo-V2's per-head softmax sink logits: tiny, and a logit.
        || name.ends_with(".self_attn.sinks")
        // MiMo audio tokenizer RVQ codebooks: nearest-neighbour tables,
        // F32 in the source; a nudged codeword changes codes.
        || name.ends_with("._codebook.embed")
        || name.ends_with(".mlp.expert_bias")
        || name.ends_with(".mlp.expert_bias_vl")
        || name.ends_with(".ffn.gate.bias")
        || name.ends_with(".ffn.gate.bias_vl")
}

/// DeepSeek-V4.1's vision tower and aligner are small relative to the text
/// checkpoint but unusually sensitive to matrix quantization. Keep their
/// matrix weights exact in the q4tp profile. The architecture guard is
/// deliberate: other model families may use the same generic prefixes for
/// unrelated tensors, and existing F32 controls are checked first by the
/// emitter below.
fn force_dsv41_vision_f16(arch: &ModelArch, name: &str, shape: &[usize]) -> bool {
    arch.deepseek_v41.is_some()
        && shape.len() == 2
        && name.ends_with(".weight")
        && (name.starts_with("vision.") || name.starts_with("aligner."))
}

/// Name-level "never quantize this matrix" rule, for tools that re-encode a
/// float container outside the streaming emitter (GPTQ). The arch-gated
/// vision rule is not included — it needs shapes and the arch.
pub(crate) fn keeps_float(name: &str) -> bool {
    force_f32(name) || force_f16(name) || name.starts_with("vis.")
}

fn force_f16(name: &str) -> bool {
    name.ends_with("model.encoder.embed_positions.weight")
        || name.ends_with("model.decoder.embed_positions.weight")
        // MiMo speech embeddings: 20 gather tables summed per frame, not matmuls.
        || name.starts_with("speech_embeddings.")
        || name.ends_with("linear_attn.in_proj_a.weight")
        || name.ends_with("linear_attn.in_proj_b.weight")
        // KDA (Kimi): decay/β/gate low-rank stages and conv taps are tiny
        // and sit on exp/σ paths — keep them exact.
        || name.ends_with("kda_attn.f_a_proj.weight")
        || name.ends_with("kda_attn.f_b_proj.weight")
        || name.ends_with("kda_attn.b_proj.weight")
        || name.ends_with("kda_attn.g_a_proj.weight")
        || name.ends_with("kda_attn.g_b_proj.weight")
        || name.ends_with("_conv1d.weight")
        // Gemma-3n: 4-wide AltUp coefficient mats and the low-rank /
        // per-layer-input projections sit on tanh/gelu gates.
        || name.ends_with("altup.correction_coefs.weight")
        || name.ends_with("altup.prediction_coefs.weight")
        || name.ends_with("altup.modality_router.weight")
        || name.ends_with("laurel.linear_left.weight")
        || name.ends_with("laurel.linear_right.weight")
        || name.ends_with("per_layer_input_gate.weight")
        || name.ends_with(".per_layer_projection.weight")
        || name.ends_with("mlp.gate.weight")
        || name.ends_with("shared_expert_gate.weight")
        || name.ends_with("self_attn.g_proj.weight")
        // Engram's tiny lookup-attention projections stay exact; only its
        // giant embed table is retained as raw FP8 auxiliary bytes.
        || name.ends_with(".engram.q_weight")
        || name.ends_with(".engram.k_weight")
        // Qwen3.8 hyper-connection projections sit directly on sigmoid
        // residual gates.  They are tiny relative to the experts and are
        // deliberately kept at f16 in a q4tp file.
        || (name.contains("hyper_connection")
            && (name.contains("input_mix_weight_") || name.contains("block_inject_weight")))
}

/// Qwen3.8-Flash-Next is unusually sensitive to error in its small recurrent
/// skeleton: every token traverses the same GDN/QSA projections 48 times,
/// while only ten of 512 routed experts are active. Keep the 172B memory wall
/// (experts + PLE lookup) in q4tp/q2tp, but give the always-active skeleton
/// q8_2f.
/// Its input-channel field preserves activation-outlier columns at essentially
/// the same size as q8-row.  This is the usual "K-medium" mixed-precision idea
/// expressed in CMF: the file/profile remains q4tp and grows only a few
/// percent, whereas a uniform four-bit GDN stack loses coherence after a few
/// dozen layers.
fn quant_for_tensor(arch: &ModelArch, name: &str, base: Quant) -> Quant {
    if !matches!(base, Quant::Q4TiledP | Quant::Q2TiledP) {
        return base;
    }
    // Whisper's attention scores determine token/audio alignment, and its
    // shared decoder embedding is also the output classifier. Keep those
    // planes at Q8 while quantizing the larger feed-forward matrices to q4tp.
    if arch.arch_name.eq_ignore_ascii_case("whisper") {
        let attention = (name.starts_with("model.encoder.layers.") && name.contains(".self_attn."))
            || (name.starts_with("model.decoder.layers.")
                && (name.contains(".self_attn.") || name.contains(".encoder_attn.")));
        if attention || name == "model.decoder.embed_tokens.weight" {
            return Quant::Q8_2f;
        }
    }
    let vocabulary_edges = name == "model.embed_tokens.weight" || name == "lm_head.weight";
    // Granite 4.2's 100k-token input/output tables are a quality-sensitive
    // boundary around an otherwise ordinary dense stack.  Keeping just those
    // two matrices in q8_2f costs little beside the 8B/30B bodies (and remains
    // worthwhile on 3B), while avoiding a second four-bit error at both the
    // first and final projection.  The full q8_2f profile is unchanged.
    if arch.arch_name.eq_ignore_ascii_case("granite") && vocabulary_edges {
        return Quant::Q8_2f;
    }
    // MiMo-V2 and Mellum are routed-MoE models whose expert banks dominate
    // the file, while every token always traverses the attention stack and
    // vocabulary edges.  Their q4tp profile therefore keeps the experts on
    // the requested 4-/2-bit plane and the always-active skeleton at q8_2f.
    // The router is f16 and norms are f16 by the emitter's force/shape rules
    // before this policy is consulted.
    if arch.arch_name == MIMO_V2 || arch.arch_name.eq_ignore_ascii_case("mellum") {
        return if name.contains(".mlp.experts.") {
            base
        } else {
            Quant::Q8_2f
        };
    }
    // MiniCPM-V 4.7 is a Qwen3.5 hybrid-MoE decoder behind a multimodal
    // wrapper.  Its expert bank dominates the footprint; the recurrent GDN,
    // full-attention layers and vocabulary boundaries run for every token.
    // Keep that skeleton at q8_2f, but keep the shared expert on the same
    // q4tp/q2tp plane as routed experts: the Metal/Vulkan MoE graph requires
    // a uniform expert codec and would otherwise decline the whole block.
    let qwen35_hybrid_moe = arch.moe.is_some()
        && arch
            .linear_core
            .as_ref()
            .is_some_and(|core| core.kind == "gated_delta_net")
        && (arch.arch_name.to_ascii_lowercase().starts_with("qwen3_5")
            || arch.arch_name.eq_ignore_ascii_case("minicpmv4_7"));
    if qwen35_hybrid_moe {
        return if name.contains(".mlp.experts.") || name.contains(".mlp.shared_expert.") {
            base
        } else {
            Quant::Q8_2f
        };
    }
    // The dense MiniCPM-V 4.7 1B sibling has the same recurrent Qwen3.5
    // decoder but no MoE bank to amortize a uniformly low-bit boundary. Its
    // 248k-token embedding and classifier are particularly quality-sensitive,
    // so retain the two vocabulary edges in q8_2f just as the established
    // MiniCPM5 mixed profile does.  This still permits the normal one-token
    // Metal/WGPU graph; only a multi-step on-device re-embed burst needs a
    // q4tp embedding and is deliberately not claimed by this fidelity-first
    // compact profile.
    let qwen35_dense_minicpmv47 = arch.arch_name.eq_ignore_ascii_case("minicpmv4_7")
        && arch.moe.is_none()
        && arch
            .linear_core
            .as_ref()
            .is_some_and(|core| core.kind == "gated_delta_net");
    if qwen35_dense_minicpmv47 && vocabulary_edges {
        return Quant::Q8_2f;
    }
    if arch.qwen4_exp.is_none() {
        return base;
    }
    let recurrent_skeleton = name.contains(".linear_attn.")
        || name.contains(".self_attn.")
        || (name.contains(".ple.")
            && (name.ends_with("key_proj.weight") || name.ends_with("value_proj.weight")))
        // the MTP head's two input projections: every draft traverses them
        || (name.starts_with("model.mtp.")
            && (name.ends_with("fc_embedding.weight") || name.ends_with("fc_hidden.weight")));
    if recurrent_skeleton || vocabulary_edges {
        Quant::Q8_2f
    } else {
        base
    }
}

/// `cortiq convert --mtp-sidecar` on a Qwen3.8-Flash-Next source: the walk
/// keeps `model.mtp.*` (and nothing else is expected to be there — the
/// source is the 31-tensor directory `tools/qwen4_mtp_fetch.py` builds).
pub(crate) fn qwen4_mtp_sidecar() -> bool {
    std::env::var("CMF_CONVERT_QWEN4_MTP").as_deref() == Ok("1")
}

/// User per-tensor quantization overrides (`cortiq convert --tensor-quant
/// PATTERN=QUANT`, repeatable). Checked before the profile policy for every
/// tensor the converter would quantize (2-D, not force-kept at f16/f32);
/// the FIRST matching pattern wins. Patterns are matched against canonical
/// tensor names with `*` as the only wildcard (it spans dots), e.g.
/// `lm_head.weight=q8_2f`, `model.layers.*.mlp.down_proj.weight=q8_2f`.
/// The file keeps its profile name (the requested `--quant`); only the
/// per-tensor dtypes in the directory change, which every reader already
/// dispatches on.
static TENSOR_QUANT_OVERRIDES: Mutex<Vec<(String, Quant)>> = Mutex::new(Vec::new());

/// Parse one `PATTERN=QUANT` override.
pub(crate) fn parse_tensor_quant(spec: &str) -> anyhow::Result<(String, Quant)> {
    let (pat, q) = spec
        .rsplit_once('=')
        .ok_or_else(|| anyhow::anyhow!("--tensor-quant '{spec}': expected PATTERN=QUANT"))?;
    let pat = pat.trim();
    anyhow::ensure!(!pat.is_empty(), "--tensor-quant '{spec}': empty pattern");
    let quant = parse_quant(q.trim())?;
    anyhow::ensure!(
        !matches!(quant, Quant::Q2TiledPAffine),
        "--tensor-quant '{spec}': q2tp_affine is a whole-file profile, not a per-tensor choice"
    );
    anyhow::ensure!(
        !matches!(quant, Quant::Fp4),
        "--tensor-quant '{spec}': fp4 is a whole-file profile; the per-tensor codec is mxfp4"
    );
    Ok((pat.to_string(), quant))
}

/// Install the converter's per-tensor overrides (replaces any earlier set).
/// `--sampling`: the model's recommended sampling, recorded in
/// `header.provenance.sampling`; `run` and `serve` start from it.
static SAMPLING_DEFAULTS: Mutex<Option<serde_json::Value>> = Mutex::new(None);

/// Parse `key=value,key=value` over [`cortiq_engine::sampler::FILE_SAMPLING_KEYS`].
pub(crate) fn parse_sampling(spec: &str) -> anyhow::Result<serde_json::Value> {
    let mut m = serde_json::Map::new();
    for part in spec.split(',').map(str::trim).filter(|s| !s.is_empty()) {
        let (k, v) = part
            .split_once('=')
            .ok_or_else(|| anyhow::anyhow!("--sampling '{part}': expected KEY=VALUE"))?;
        let k = k.trim();
        anyhow::ensure!(
            cortiq_engine::sampler::FILE_SAMPLING_KEYS.contains(&k),
            "--sampling: unknown key '{k}' (known: {})",
            cortiq_engine::sampler::FILE_SAMPLING_KEYS.join(", ")
        );
        let v = v.trim();
        let val = if k == "top_k" || k == "max_tokens" {
            serde_json::json!(v
                .parse::<u32>()
                .map_err(|e| anyhow::anyhow!("--sampling {k} '{v}': {e}"))?)
        } else {
            let x = v
                .parse::<f64>()
                .map_err(|e| anyhow::anyhow!("--sampling {k} '{v}': {e}"))?;
            anyhow::ensure!(x.is_finite() && x >= 0.0, "--sampling {k}: {x} must be finite and >= 0");
            serde_json::json!(x)
        };
        m.insert(k.to_string(), val);
    }
    anyhow::ensure!(!m.is_empty(), "--sampling: nothing to record");
    Ok(serde_json::Value::Object(m))
}

pub fn set_sampling_defaults(spec: Option<&str>) -> anyhow::Result<()> {
    *SAMPLING_DEFAULTS.lock().unwrap() = spec.map(parse_sampling).transpose()?;
    Ok(())
}

fn sampling_defaults() -> Option<serde_json::Value> {
    SAMPLING_DEFAULTS.lock().unwrap().clone()
}

pub fn set_tensor_quant_overrides(specs: &[String]) -> anyhow::Result<()> {
    let parsed = specs
        .iter()
        .map(|s| parse_tensor_quant(s))
        .collect::<anyhow::Result<Vec<_>>>()?;
    *TENSOR_QUANT_OVERRIDES.lock().unwrap() = parsed;
    Ok(())
}

/// `*`-glob match (the only wildcard; it matches any run, dots included).
pub(crate) fn glob_match(pat: &str, name: &str) -> bool {
    let (p, n) = (pat.as_bytes(), name.as_bytes());
    let (mut pi, mut ni) = (0usize, 0usize);
    let (mut star, mut mark) = (None::<usize>, 0usize);
    while ni < n.len() {
        if pi < p.len() && p[pi] == b'*' {
            star = Some(pi);
            mark = ni;
            pi += 1;
        } else if pi < p.len() && p[pi] == n[ni] {
            pi += 1;
            ni += 1;
        } else if let Some(s) = star {
            pi = s + 1;
            mark += 1;
            ni = mark;
        } else {
            return false;
        }
    }
    while pi < p.len() && p[pi] == b'*' {
        pi += 1;
    }
    pi == p.len()
}

pub(crate) fn tensor_quant_override(name: &str) -> Option<Quant> {
    let o = TENSOR_QUANT_OVERRIDES.lock().unwrap();
    o.iter().find(|(p, _)| glob_match(p, name)).map(|&(_, q)| q)
}

/// The quant one 2-D tensor gets: a user override, else the profile policy.
pub(crate) fn effective_quant(arch: &ModelArch, requested: Quant, name: &str) -> Quant {
    tensor_quant_override(name).unwrap_or_else(|| profile_quant(arch, requested, name))
}

/// Parse `cortiq convert --exclude-tensors` values: each one a
/// comma-separated list of globs over CANONICAL output tensor names (the
/// `--tensor-quant` glob: `*` is the only wildcard and spans dots).
/// Duplicates collapse; an empty pattern is refused.
pub fn parse_tensor_exclusions(specs: &[String]) -> anyhow::Result<Vec<String>> {
    let mut out: Vec<String> = Vec::new();
    for spec in specs {
        for p in spec.split(',') {
            let p = p.trim();
            anyhow::ensure!(
                !p.is_empty(),
                "--exclude-tensors '{spec}': empty pattern (expected comma-separated globs, e.g. 'model.mtp.*')"
            );
            if !out.iter().any(|q| q == p) {
                out.push(p.to_string());
            }
        }
    }
    Ok(out)
}

/// `cortiq convert --exclude-tensors PATTERNS`: tensors left out of every
/// output of a conversion. A source tensor whose canonical name matches is
/// skipped before its payload is read — no decode, no requant, no write —
/// and counted (tensors and source bytes). A name only a split produces (one
/// expert of a packed tensor, q/k/v of a fused projection) is also checked
/// when it reaches the writer, so a pattern over such names holds too; that
/// source tensor was decoded for its other parts. The patterns are recorded
/// in `header.provenance.excluded_tensors`.
///
/// Patterns are matched twice: against the canonical SOURCE name (after the
/// architecture's canonical renames, before the later per-family ones —
/// Kimi `.weight_packed` → `.weight`, Gemma-4 `router.proj` →
/// `mlp.gate`, the MiMo/Phi qkv and gate_up splits) and against the final
/// OUTPUT name at the writer. A match on the output name alone drops the
/// tensor after its source was decoded; a match on a pre-rename source name
/// drops the tensor even though its output name differs. Gemma-4
/// `.router.scale` tensors are read in a pre-pass whatever the patterns.
/// None of this concerns DeepSeek-V4.1, whose `mtp.N.*` names become
/// `model.mtp.N.*` in the canonical source pass.
///
/// A resumed output must keep the patterns it was started with: they are
/// recorded next to its manifest (`<output>.manifest.exclude`) and a resume
/// with other patterns, or with patterns that match a tensor the output
/// already holds, is refused.
///
/// DeepSeek-V4.1-Flash-1017 with `--quant fp4` is 504.51 GB with its
/// MTP/DSpark stack and 496.96 GB without; `model.mtp.*` drops that stack,
/// which the V4.1 runtime never executes, and brings the file under Hugging
/// Face's 500 GB per-file limit.
pub(crate) struct TensorExclusion {
    patterns: Vec<String>,
    /// Source tensors excluded per pattern (first match).
    hits: Vec<std::cell::Cell<usize>>,
    source_tensors: std::cell::Cell<usize>,
    source_bytes: std::cell::Cell<u64>,
    /// Split-produced names dropped at the writer (unique names).
    derived: std::cell::RefCell<HashSet<String>>,
}

impl TensorExclusion {
    pub(crate) fn new(patterns: &[String]) -> Self {
        Self {
            patterns: patterns.to_vec(),
            hits: patterns.iter().map(|_| std::cell::Cell::new(0)).collect(),
            source_tensors: std::cell::Cell::new(0),
            source_bytes: std::cell::Cell::new(0),
            derived: std::cell::RefCell::new(HashSet::new()),
        }
    }

    pub(crate) fn is_empty(&self) -> bool {
        self.patterns.is_empty()
    }

    fn matching(&self, name: &str) -> Option<usize> {
        self.patterns.iter().position(|p| glob_match(p, name))
    }

    /// Whether the source tensor with canonical `name` (payload `bytes`
    /// long) is excluded; counts it if so.
    fn skip_source(&self, name: &str, bytes: usize) -> bool {
        let Some(i) = self.matching(name) else {
            return false;
        };
        self.hits[i].set(self.hits[i].get() + 1);
        self.source_tensors.set(self.source_tensors.get() + 1);
        self.source_bytes
            .set(self.source_bytes.get().saturating_add(bytes as u64));
        true
    }

    /// Whether an output tensor about to be written is excluded (a name the
    /// source-side check could not see).
    fn skip_output(&self, name: &str) -> bool {
        if self.patterns.is_empty() || self.matching(name).is_none() {
            return false;
        }
        self.derived.borrow_mut().insert(name.to_string());
        true
    }

    /// Before any shard is read: how many tensors of the source index each
    /// pattern matches (canonical names), and a warning for a pattern that
    /// matches none — a typo shows in the first seconds, not after a
    /// multi-hour conversion. Such a pattern may still match a name only a
    /// split produces, so it is not refused.
    fn preview(&self, names: &[String]) {
        for line in self.preview_lines(names) {
            eprintln!("{line}");
        }
    }

    fn preview_lines(&self, names: &[String]) -> Vec<String> {
        if self.patterns.is_empty() || names.is_empty() {
            return Vec::new();
        }
        let mut per = vec![0usize; self.patterns.len()];
        for n in names {
            if let Some(i) = self.matching(n) {
                per[i] += 1;
            }
        }
        let mut out = vec![format!(
            "  --exclude-tensors: the source index lists {} matching tensors ({})",
            per.iter().sum::<usize>(),
            self.patterns
                .iter()
                .zip(&per)
                .map(|(p, k)| format!("'{p}' ×{k}"))
                .collect::<Vec<_>>()
                .join(", ")
        )];
        for (p, k) in self.patterns.iter().zip(&per) {
            if *k == 0 {
                out.push(format!(
                    "  warning: --exclude-tensors '{p}' matches no tensor of the source index \
                     (patterns match canonical CMF names, e.g. 'model.mtp.*'); only a name a \
                     split produces at write time can still match it"
                ));
            }
        }
        out
    }

    /// The end-of-run summary. `shards_done_before`: source shards earlier
    /// attempts of a resumed output had already converted — their tensors
    /// were not re-read, so they are not in the counts.
    fn report(&self, shards_done_before: usize) {
        for line in self.report_lines(shards_done_before) {
            eprintln!("{line}");
        }
    }

    fn report_lines(&self, shards_done_before: usize) -> Vec<String> {
        if self.patterns.is_empty() {
            return Vec::new();
        }
        let per: Vec<String> = self
            .patterns
            .iter()
            .zip(&self.hits)
            .map(|(p, h)| format!("'{p}' ×{}", h.get()))
            .collect();
        let mut out = vec![format!(
            "  --exclude-tensors: {} source tensors excluded, {} source bytes ({:.2} GB; {}){}{}",
            self.source_tensors.get(),
            self.source_bytes.get(),
            self.source_bytes.get() as f64 / 1e9,
            per.join(", "),
            match self.derived.borrow().len() {
                0 => String::new(),
                n => format!("; {n} split-produced tensors dropped at write"),
            },
            match shards_done_before {
                0 => String::new(),
                n => format!(
                    " — resumed: these count only the shards this run converted; {n} source \
                     shards earlier attempts finished were not re-read"
                ),
            }
        )];
        for (p, h) in self.patterns.iter().zip(&self.hits) {
            if h.get() == 0 && !self.derived.borrow().iter().any(|n| glob_match(p, n)) {
                out.push(if shards_done_before == 0 {
                    format!(
                        "  warning: --exclude-tensors '{p}' matched no tensor in this run \
                         (patterns match canonical CMF names, e.g. 'model.mtp.*')"
                    )
                } else {
                    format!(
                        "  note: --exclude-tensors '{p}' matched no tensor in the shards this \
                         run converted (its tensors may sit in the {shards_done_before} shards \
                         finished earlier)"
                    )
                });
            }
        }
        out
    }
}

/// Where a conversion records its `--exclude-tensors` patterns next to an
/// output's resume manifest.
fn exclusion_record_path(manifest_path: &str) -> String {
    format!("{manifest_path}.exclude")
}

/// A new output records its exclusion patterns next to its manifest (none:
/// a stale record from an earlier output of the same name is removed), so
/// a resume can check it continues with the same ones.
fn record_exclusion(manifest_path: &str, patterns: &[String]) -> std::io::Result<()> {
    let path = exclusion_record_path(manifest_path);
    if patterns.is_empty() {
        return match fs::remove_file(&path) {
            Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e),
            _ => Ok(()),
        };
    }
    fs::write(&path, serde_json::to_vec(patterns).map_err(std::io::Error::other)?)
}

/// A resumed output keeps the exclusion it was started with: shards done
/// earlier were written under those patterns, and a different set would
/// leave the file holding tensors its provenance says are excluded, or
/// missing tensors its provenance does not mention. The record next to the
/// manifest is compared when present; an output started before records
/// existed (or without patterns) only has to hold no tensor the patterns
/// match, and then gets a record.
fn check_resume_exclusion(
    output_path: &str,
    manifest_path: &str,
    patterns: &[String],
    exclusion: &TensorExclusion,
    written: &[String],
) -> anyhow::Result<()> {
    let shown = |p: &[String]| {
        if p.is_empty() {
            "none".to_string()
        } else {
            format!("'{}'", p.join(","))
        }
    };
    let record = exclusion_record_path(manifest_path);
    let recorded: Option<Vec<String>> = match fs::read(&record) {
        Ok(b) => Some(serde_json::from_slice(&b).map_err(|e| {
            anyhow::anyhow!("resume {output_path}: unreadable exclusion record {record}: {e}")
        })?),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
        Err(e) => anyhow::bail!("resume {output_path}: read {record}: {e}"),
    };
    if let Some(rec) = recorded {
        let set = |p: &[String]| p.iter().cloned().collect::<std::collections::BTreeSet<_>>();
        anyhow::ensure!(
            set(&rec) == set(patterns),
            "resume {output_path}: the output was started with --exclude-tensors {} and is \
             resumed with {}; resume with the same patterns or start a new output",
            shown(&rec),
            shown(patterns)
        );
        return Ok(());
    }
    if let Some((name, p)) = written
        .iter()
        .find_map(|n| exclusion.matching(n).map(|i| (n, &exclusion.patterns[i])))
    {
        anyhow::bail!(
            "resume {output_path}: the output was started without --exclude-tensors and already \
             holds '{name}', which '{p}' excludes; resume without the flag or start a new output"
        );
    }
    record_exclusion(manifest_path, patterns)
        .map_err(|e| anyhow::anyhow!("resume {output_path}: {e}"))
}

/// Quantization choice for 2-D weight matrices.
#[derive(Clone, Copy, PartialEq, Debug)]
pub(crate) enum Quant {
    Q8Row,
    Q8_2f,
    Q4Block,
    F16,
    /// Grouped variable-bit (per-row 3–8 bit, water-filled by row amplitude).
    Vbit,
    Q4Tiled,
    /// q4 tiled with PREDICTED scales: same nibbles, but the per-tile f16
    /// scale becomes a 5-bit rung on a per-row ladder — 7.3% off a q4t file
    /// for 1.14% RMS weight perturbation (q4t's own error vs fp16 is ~10%).
    Q4TiledP,
    /// 2-bit tiles on the q4tp ladder (Escha-W2 size class). As a whole-model
    /// profile the converter applies it to MoE `gate_up` experts only and
    /// keeps `down` experts and the skeleton at q4tp — the 2/4 split that
    /// mirrors Escha's 2/3-bit choice.
    Q2TiledP,
    /// Existing dtype16 Q2TP payload with the explicit Prism affine operator
    /// `(c - 1.0) * s` enabled by header metadata.
    Q2TiledPAffine,
    /// 1-bit binary (explicit opt-in): for 1-bit-TRAINED models
    /// (Bonsai / BitNet class), where per-group weights already sit on
    /// two levels ±s and the encoding is (near-)lossless. As PTQ of a
    /// normal checkpoint this destroys quality — never a default.
    Q1,
    /// 1-bit PTQ of a NORMAL checkpoint via error diffusion (перетекание):
    /// same on-disk `Q1` tile, but the encoder carries each weight's sign
    /// residual forward so the row sum survives binarization. Training-free;
    /// pair with `cortiq skill bake` (FCD) on the tail layers to recover
    /// quality. Bit-identical to `q1` on a genuinely 1-bit model.
    Q1p,
    /// 1-bit PTQ with an outlier mask (`Q1S` dtype): keeps the heavy tail
    /// (`CMF_Q1S_KEEP` of weights by |value|, default 1%) at full f16 in a
    /// sparse overlay, binarizes the rest with error diffusion. The mask
    /// lever of the error-feedback path — what lets a NORMAL
    /// checkpoint survive 1-bit.
    Q1s,
    Q1t,
    /// OCP MX FP4 (`mxfp4` dtype 21): E2M1 codes + one E8M0 scale per 32
    /// elements, 4.25 bits/weight. As a per-tensor codec (`--tensor-quant
    /// PATTERN=mxfp4`) or a whole-file `--quant mxfp4`, float sources are
    /// quantized by the OCP rule; an OCP MXFP4 source tensor (DeepSeek-V4/
    /// V4.1 `I8` experts + `F8_E8M0` 1×32 scales) is re-packed byte for
    /// byte instead.
    Mxfp4,
    /// `--quant fp4`: the native-FP4 MoE profile. Every routed and shared
    /// expert matrix (gate/up/down) is `mxfp4` — byte-identical to an OCP
    /// MXFP4 source, OCP-quantized from any other source so the expert bank
    /// keeps one codec — and every other tensor gets exactly what the
    /// `q4tp` profile gives it.
    Fp4,
}

/// Whether `name` is a routed or shared expert's gate/up/down matrix — the
/// tensors the `fp4` profile stores as `mxfp4`.
pub(crate) fn fp4_expert_matrix(name: &str) -> bool {
    let expert = name.contains(".experts.")
        || name.contains(".shared_expert.")
        || name.contains(".shared_experts.");
    expert
        && [
            ".gate_proj.weight",
            ".up_proj.weight",
            ".down_proj.weight",
            ".w1.weight",
            ".w2.weight",
            ".w3.weight",
        ]
        .iter()
        .any(|s| name.ends_with(s))
}

/// Quantize a 2-D matrix `[out_dim, in_dim]` per the chosen scheme.
pub(crate) fn quantize_2d(
    quant: Quant,
    vals: &[f32],
    out_dim: usize,
    in_dim: usize,
) -> (TensorDtype, Vec<u8>) {
    match quant {
        Quant::Q8Row => (TensorDtype::Q8Row, encode_q8_row(vals, out_dim, in_dim)),
        Quant::Q8_2f => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Q4Block => (TensorDtype::Q4Block, encode_q4_block(vals)),
        Quant::F16 => (TensorDtype::F16, encode_f16(vals)),
        // v-bit needs the input dim to be a multiple of the group size; other
        // shapes fall back to the two-field q8_2f (best equal-size alternative).
        Quant::Q4Tiled if in_dim % GROUP_SIZE == 0 => {
            (TensorDtype::Q4Tiled, encode_q4_tiled(vals, out_dim, in_dim))
        }
        Quant::Q4Tiled => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Q4TiledP if in_dim % GROUP_SIZE == 0 => {
            (TensorDtype::Q4TiledP, encode_q4tp(vals, out_dim, in_dim))
        }
        Quant::Q4TiledP => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Q2TiledP if in_dim % GROUP_SIZE == 0 => {
            (TensorDtype::Q2TiledP, encode_q2tp(vals, out_dim, in_dim))
        }
        Quant::Q2TiledP => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Q2TiledPAffine if in_dim % GROUP_SIZE == 0 => {
            (TensorDtype::Q2TiledP, encode_q2tp(vals, out_dim, in_dim))
        }
        Quant::Q2TiledPAffine => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Vbit if in_dim % GROUP_SIZE == 0 => {
            (TensorDtype::VbitRo, encode_vbit_ro(vals, out_dim, in_dim))
        }
        Quant::Vbit => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Q1 if in_dim % GROUP_SIZE == 0 => {
            (TensorDtype::Q1, encode_q1(vals, out_dim, in_dim))
        }
        Quant::Q1 => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Q1p if in_dim % GROUP_SIZE == 0 => {
            (TensorDtype::Q1, encode_q1_ef(vals, out_dim, in_dim))
        }
        Quant::Q1p => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Q1s if in_dim % GROUP_SIZE == 0 => (
            TensorDtype::Q1S,
            encode_q1s(vals, out_dim, in_dim, q1s_keep_frac()),
        ),
        Quant::Q1s => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Q1t if in_dim % GROUP_SIZE == 0 => (
            TensorDtype::Q1T,
            crate::gptq::quantize_q1t(vals, out_dim, in_dim, &vec![1.0; in_dim], 0.0),
        ),
        Quant::Q1t => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        Quant::Mxfp4 if in_dim % GROUP_SIZE == 0 => (
            TensorDtype::Mxfp4,
            cortiq_core::mxfp4::quantize(vals, out_dim, in_dim),
        ),
        Quant::Mxfp4 => (TensorDtype::Q8_2f, encode_q8_2f(vals, out_dim, in_dim)),
        // The profile resolves per tensor (`profile_quant`); a direct caller
        // gets the profile's expert codec.
        Quant::Fp4 => quantize_2d(Quant::Mxfp4, vals, out_dim, in_dim),
    }
}

pub(crate) fn parse_quant(s: &str) -> anyhow::Result<Quant> {
    Ok(match s.to_ascii_lowercase().as_str() {
        "q8" | "q8_row" | "q8row" => Quant::Q8Row,
        "q8_2f" | "q82f" | "q8f" => Quant::Q8_2f,
        "q4" | "q4_block" | "q4block" => Quant::Q4Block,
        "f16" | "fp16" => Quant::F16,
        "vbit" | "v_bit" => Quant::Vbit,
        "q4t" | "q4_tiled" => Quant::Q4Tiled,
        "q4tp" | "q4t_pred" => Quant::Q4TiledP,
        "q2tp" | "q2t_pred" => Quant::Q2TiledP,
        "q2tp_affine" | "q2tp-affine" | "q2t_affine" => Quant::Q2TiledPAffine,
        "q1" => Quant::Q1,
        "q1p" | "q1_ptq" => Quant::Q1p,
        "q1s" | "q1_mask" => Quant::Q1s,
        "q1t" | "q1_ternary" => Quant::Q1t,
        "mxfp4" | "mx_fp4" => Quant::Mxfp4,
        "fp4" => Quant::Fp4,
        other => anyhow::bail!(
            "unknown quant '{other}' (use q8, q8_2f, q4, q4t, q4tp, q2tp, q2tp_affine, f16, vbit, q1, q1p, q1s, q1t, mxfp4, or fp4)"
        ),
    })
}

pub(crate) fn quant_name(quant: Quant) -> &'static str {
    match quant {
        Quant::Q8Row => "q8",
        Quant::Q8_2f => "q8_2f",
        Quant::Q4Block => "q4",
        Quant::F16 => "f16",
        Quant::Vbit => "vbit",
        Quant::Q4Tiled => "q4t",
        Quant::Q4TiledP => "q4tp",
        Quant::Q2TiledP => "q2tp",
        Quant::Q2TiledPAffine => "q2tp_affine",
        Quant::Q1 => "q1",
        Quant::Q1p => "q1p",
        Quant::Q1s => "q1s",
        Quant::Q1t => "q1t",
        Quant::Mxfp4 => "mxfp4",
        Quant::Fp4 => "fp4",
    }
}

/// Quantization profile used by the converter's one-pass multi-output path.
/// `Q2TiledP` is a logical profile: the file keeps the normal q4tp layout for
/// the skeleton/down planes while putting gate/up expert (or dense FFN) planes
/// on q2tp. Keeping this decision here means every requested output sees the
/// same decoded source tensor and only the encoder differs.
fn profile_quant(arch: &ModelArch, requested: Quant, name: &str) -> Quant {
    if matches!(requested, Quant::Fp4) {
        // Experts on the native FP4 codec, everything else exactly as the
        // q4tp profile places it (skeleton q8_2f rules included).
        return if fp4_expert_matrix(name) {
            Quant::Mxfp4
        } else {
            profile_quant(arch, Quant::Q4TiledP, name)
        };
    }
    if matches!(requested, Quant::Q2TiledPAffine) {
        // Prism weights are all trained ternary projections in one rotated
        // basis; never apply q2tp's mixed expert-only policy.
        return requested;
    }
    let gu = requested;
    let base = if matches!(requested, Quant::Q2TiledP) {
        Quant::Q4TiledP
    } else {
        requested
    };
    let expert_gu = q2tp_expert_gate_or_up(name)
        || (matches!(requested, Quant::Q2TiledP)
            && arch.moe.is_none()
            && q2tp_dense_gate_or_up(name));
    let q = if expert_gu { gu } else { base };
    quant_for_tensor(arch, name, q)
}

#[derive(Clone, Copy)]
enum EmitMode {
    /// Apply the normal per-name profile policy (including q2tp gate/up).
    Auto,
    /// Keep this tensor in exact f16 regardless of profile.
    F16,
    /// Keep this tensor in exact f32 regardless of profile.
    F32,
}

/// Encode one canonical tensor into each active output profile. The source
/// values are borrowed and are therefore decoded exactly once per source
/// tensor; each profile owns its independent encoded payload and directory
/// record. `active` is false for outputs that already checkpointed this shard.
fn emit_profiled_tensor(
    batches: &mut [Vec<TensorSpec>],
    profiles: &[Quant],
    active: &[bool],
    arch: &ModelArch,
    name: &str,
    shape: &[usize],
    vals: &[f32],
    mode: EmitMode,
) {
    debug_assert_eq!(batches.len(), profiles.len());
    debug_assert_eq!(active.len(), profiles.len());
    let numel: usize = shape.iter().product();
    for ((batch, &requested), &is_active) in batches.iter_mut().zip(profiles).zip(active) {
        if !is_active {
            continue;
        }
        let (dtype, data) = match mode {
            EmitMode::F32 => (
                TensorDtype::F32,
                vals.iter().flat_map(|v| v.to_le_bytes()).collect(),
            ),
            EmitMode::F16 => (TensorDtype::F16, encode_f16(vals)),
            EmitMode::Auto => {
                if force_f32(name) {
                    (
                        TensorDtype::F32,
                        vals.iter().flat_map(|v| v.to_le_bytes()).collect(),
                    )
                } else if force_dsv41_vision_f16(arch, name, shape) {
                    (TensorDtype::F16, encode_f16(vals))
                } else if shape.len() != 2
                    || numel < GROUP_SIZE
                    || force_f16(name)
                    // Prism's Qwen3 vision tower is retained losslessly.  Its
                    // qwen3vis runtime expects f16/f32 matrices, not the
                    // language-side q2tp transform.
                    || name.starts_with("vis.")
                {
                    (TensorDtype::F16, encode_f16(vals))
                } else {
                    quantize_2d(
                        effective_quant(arch, requested, name),
                        vals,
                        shape[0],
                        shape[1],
                    )
                }
            }
        };
        batch.push(TensorSpec {
            name: name.to_string(),
            dtype,
            shape: shape.to_vec(),
            data,
        });
    }
}

/// q8_row: `[int8 : out·in][f16 : out]` (validated layout, matches the reader).
pub(crate) fn encode_q8_row(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    let mut q = Vec::with_capacity(out_dim * in_dim);
    let mut scales = Vec::with_capacity(out_dim * 2);
    for o in 0..out_dim {
        let row = &vals[o * in_dim..(o + 1) * in_dim];
        let absmax = row.iter().fold(0f32, |m, v| m.max(v.abs()));
        let scale = f16_scale(absmax / 127.0);
        for &v in row {
            // round-half-to-even matches numpy's np.round → byte-identical weights.
            q.push((v / scale).round_ties_even().clamp(-128.0, 127.0) as i8 as u8);
        }
        scales.extend_from_slice(&f32_to_f16(scale).to_le_bytes());
    }
    q.extend_from_slice(&scales);
    q
}

/// q4_block: groups of 32 over the flattened tensor, `[u8 packed][f16 scales]`.
fn encode_q4_block(vals: &[f32]) -> Vec<u8> {
    let n_groups = vals.len().div_ceil(GROUP_SIZE);
    let mut padded = vals.to_vec();
    padded.resize(n_groups * GROUP_SIZE, 0.0);
    let mut packed = Vec::with_capacity(n_groups * 16);
    let mut scales = Vec::with_capacity(n_groups * 2);
    for g in 0..n_groups {
        let group = &padded[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
        let absmax = group.iter().fold(0f32, |m, v| m.max(v.abs()));
        let scale = f16_scale(absmax / 7.0);
        for k in 0..16 {
            let q0 = ((group[k * 2] / scale).round_ties_even().clamp(-8.0, 7.0) as i8 + 8) as u8;
            let q1 = ((group[k * 2 + 1] / scale)
                .round_ties_even()
                .clamp(-8.0, 7.0) as i8
                + 8) as u8;
            packed.push((q0 & 0x0F) | (q1 << 4));
        }
        scales.extend_from_slice(&f32_to_f16(scale).to_le_bytes());
    }
    packed.extend_from_slice(&scales);
    packed
}

/// q8_2f (row/column-scale): `[int8: out·in][f16 row_scale: out][f16 col: in]`.
/// `col[i]` = RMS over rows (absorbs outlier input channels); each row is int8
/// over the residual normalized by col. Dequant: `w = q·scale[o]·col[i]`.
/// Recovers most of the q8→f16 quality gap at the same size.
/// q4_tiled (§4.3): the same 4-bit values/scales as q4_block, laid
/// out as one sequential stream of 18-byte tiles
/// `[f16 scale][16B nibbles]` per 32-group — measured x1.66 (ARM) /
/// x1.13 (AVX2) at kernel level over the split layout.
pub(crate) fn encode_q4_tiled(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    debug_assert_eq!(vals.len(), out_dim * in_dim);
    debug_assert_eq!(in_dim % GROUP_SIZE, 0);
    let legacy = encode_q4_block(vals);
    let n_groups = vals.len() / GROUP_SIZE;
    let (packed, scales) = legacy.split_at(n_groups * 16);
    let mut out = Vec::with_capacity(n_groups * 18);
    for g in 0..n_groups {
        out.extend_from_slice(&scales[g * 2..g * 2 + 2]);
        out.extend_from_slice(&packed[g * 16..(g + 1) * 16]);
    }
    out
}

/// q4tp (§4.10): the same nibbles as `q4_tiled`, but each tile's scale is a
/// 5-bit rung on a per-row geometric ladder — `[nibbles][row (f16 lo, f16
/// step)][5-bit codes]`, 4.17 bits/weight against 4.50.
///
/// Two details carry the accuracy. The row's `lo`/`step` are rounded to f16
/// FIRST and the codes are then chosen against those rounded values, so the
/// rounding of `lo` is absorbed by the code instead of stacking on top of it.
/// And the nibbles are quantized against the RECONSTRUCTED scale, not the
/// exact per-tile absmax — otherwise encoder and reader disagree, which is
/// the same trap `f16_scale` exists to avoid.
///
/// Quantizing the same fp32 weights both ways, q4tp's error against the source
/// is within 0.1% of q4t's at the median within-row scale spread measured on
/// KAT-Coder-V2.5 (1.27 in log2), 0.3% at its 90th percentile — a coarser scale
/// costs almost nothing because the nibbles re-round against it. Rounding the
/// code to nearest beats rounding up (1.19% vs 1.60% RMS against q4t's output):
/// letting the top tile clip a nibble costs less than coarsening the whole row.
/// `q2tp`: the `q4tp` ladder with a 2-bit weight plane (8 B per 32-group).
/// Levels are (c − 1.5)·s, so the group scale fits absmax/1.5. Built for
/// transcoding 2-bit-class checkpoints (Escha-W2) where the source already
/// paid the big quantization cost — and for MoE experts generally.
/// How many threads the row encoders use. `CMF_ENCODE_THREADS` overrides it,
/// which is also how the parity test pins a single thread.
pub(crate) fn encode_threads() -> usize {
    std::env::var("CMF_ENCODE_THREADS")
        .ok()
        .and_then(|v| v.parse::<usize>().ok())
        .filter(|v| *v > 0)
        .unwrap_or_else(|| {
            std::thread::available_parallelism()
                .map(|v| v.get())
                .unwrap_or(1)
        })
}

/// Run a per-row encoder across the machine's cores.
///
/// Every tiled layout here writes three planes indexed by row at a fixed
/// stride, and no row reads another, so splitting on rows is exact — the
/// bytes are identical to the serial loop, which the parity test pins.
///
/// It is worth the plumbing: quantizing a 300B MoE is hours of arithmetic,
/// and on a 48-core box the serial loop used one of them.
fn encode_rows_parallel(
    out_dim: usize,
    strides: [usize; 3],
    planes: [&mut [u8]; 3],
    row: &(dyn Fn(usize, &mut [u8], &mut [u8], &mut [u8]) + Sync),
) {
    let [s0, s1, s2] = strides;
    let [p0, p1, p2] = planes;
    let threads = encode_threads().min(out_dim.max(1)).max(1);
    if threads == 1 {
        for (i, ((a, b), c)) in p0
            .chunks_mut(s0)
            .zip(p1.chunks_mut(s1))
            .zip(p2.chunks_mut(s2))
            .enumerate()
        {
            row(i, a, b, c);
        }
        return;
    }
    let per = out_dim.div_ceil(threads);
    std::thread::scope(|sc| {
        for (ti, ((a, b), c)) in p0
            .chunks_mut(per * s0)
            .zip(p1.chunks_mut(per * s1))
            .zip(p2.chunks_mut(per * s2))
            .enumerate()
        {
            sc.spawn(move || {
                let r0 = ti * per;
                for (i, ((aa, bb), cc)) in a
                    .chunks_mut(s0)
                    .zip(b.chunks_mut(s1))
                    .zip(c.chunks_mut(s2))
                    .enumerate()
                {
                    row(r0 + i, aa, bb, cc);
                }
            });
        }
    });
}

/// The rung search can be turned off (`CMF_Q2TP_SEARCH=0`) so its value is
/// measured rather than assumed. Read once per tensor — not cached for the
/// process, or an A/B inside one test binary silently compares an arm with
/// itself.
fn q2tp_search() -> bool {
    std::env::var("CMF_Q2TP_SEARCH")
        .map(|v| v != "0")
        .unwrap_or(true)
}

/// `CMF_Q2TP_ENCODER=v1` restores the absmax-pinned q2tp encoder byte for
/// byte; anything else (or nothing) runs v2. Read per tensor for the same
/// reason as [`q2tp_search`].
fn q2tp_encoder_is_v1() -> bool {
    std::env::var("CMF_Q2TP_ENCODER")
        .map(|v| v.trim().eq_ignore_ascii_case("v1"))
        .unwrap_or(false)
}

/// Encode a `[out_dim, in_dim]` matrix as `q2tp`. The default is the v2
/// encoder ([`encode_q2tp_v2`]); `CMF_Q2TP_ENCODER=v1` selects the old one,
/// and `CMF_Q2TP_SEARCH=0` the plain absmax rule with no search at all.
/// Both write the same layout for the same reader and kernels.
pub(crate) fn encode_q2tp(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    let search = q2tp_search();
    if search && !q2tp_encoder_is_v1() {
        encode_q2tp_v2(vals, out_dim, in_dim)
    } else {
        encode_q2tp_v1(vals, out_dim, in_dim, search)
    }
}

/// The v1 `q2tp` encoder: the outer level pinned to absmax, rungs searched
/// over nominal−2..+1, ladder top over four fractions. Kept verbatim — it is
/// also the "cortiq rung rule" ladder `quantize-gptq --codec q2tp` starts from.
pub(crate) fn encode_q2tp_v1(vals: &[f32], out_dim: usize, in_dim: usize, search: bool) -> Vec<u8> {
    debug_assert_eq!(vals.len(), out_dim * in_dim);
    debug_assert_eq!(in_dim % GROUP_SIZE, 0);
    let gpr = in_dim / GROUP_SIZE;
    let stride = q4tp_code_stride(gpr);
    let mut chunks = vec![0u8; out_dim * gpr * Q2TP_CHUNK];
    let mut params = vec![0u8; out_dim * 4];
    let mut codes = vec![0u8; out_dim * stride];

    encode_rows_parallel(
        out_dim,
        [gpr * Q2TP_CHUNK, 4, stride],
        [&mut chunks, &mut params, &mut codes],
        &|r, chunks_row, params_row, codes_row| {
            let mut lg = vec![0f32; gpr];
            let mut dead = vec![false; gpr];

            let row = &vals[r * in_dim..(r + 1) * in_dim];
            let (mut lo, mut hi) = (f32::INFINITY, f32::NEG_INFINITY);
            for g in 0..gpr {
                let tile = &row[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
                let absmax = tile.iter().fold(0f32, |m, v| m.max(v.abs()));
                dead[g] = absmax == 0.0;
                lg[g] = det_log2(f16_scale(absmax / 1.5));
                if !dead[g] {
                    lo = lo.min(lg[g]);
                    hi = hi.max(lg[g]);
                }
            }
            if !lo.is_finite() {
                lo = lg[0];
                hi = lo;
            }
            let lo_h = f32_to_f16(lo);
            let lo_r = f16_to_f32(lo_h);
            // The ladder spans lo..hi, where hi is the largest group scale in the
            // row. One loud group therefore stretches the ladder and coarsens the
            // step for all the quiet ones. Try shorter ladders too and keep the
            // one whose whole row reconstructs best — the same argument as the
            // per-group search, one level up.
            let ladder_for = |top: f32, params: &mut [u8]| -> [f32; 32] {
                let span = (top - lo_r).max(0.0);
                let mut st_h = f32_to_f16(span / Q2TP_LMAX as f32);
                for _ in 0..64 {
                    let st = f16_to_f32(st_h);
                    if st > 0.0 && lo_r + Q2TP_LMAX as f32 * st >= top {
                        break;
                    }
                    st_h += 1;
                }
                params[0..2].copy_from_slice(&lo_h.to_le_bytes());
                params[2..4].copy_from_slice(&st_h.to_le_bytes());
                det_q2tp_ladder(params, 0)
            };
            let mut tab = ladder_for(hi, params_row);
            if search && hi > lo_r {
                // Candidate tops: the full range, then progressively tighter.
                let mut best = (f32::INFINITY, hi);
                for frac in [1.0f32, 0.9, 0.8, 0.7] {
                    let top = lo_r + (hi - lo_r) * frac;
                    let mut probe = [0u8; 4];
                    let t = ladder_for(top, &mut probe);
                    let mut err = 0.0f32;
                    for g in 0..gpr {
                        if dead[g] {
                            continue;
                        }
                        let tile = &row[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
                        // Probe around the rung this ladder would nominally pick
                        // rather than all 31: the best is always adjacent, and
                        // scanning the ladder end to end costs 8x for nothing.
                        let stc = f16_to_f32(u16::from_le_bytes([probe[2], probe[3]]));
                        let nom = if stc > 0.0 {
                            1 + ((lg[g] - lo_r) / stc)
                                .round_ties_even()
                                .clamp(0.0, Q2TP_LMAX as f32)
                                as usize
                        } else {
                            1
                        };
                        let mut ge = f32::INFINITY;
                        for cand in nom.saturating_sub(2).max(1)..=(nom + 2).min(Q2TP_LMAX + 1) {
                            let sc = t[cand];
                            if sc <= 0.0 {
                                continue;
                            }
                            let iv = 1.0 / sc;
                            let mut e = 0.0f32;
                            for &wv in tile {
                                let q = (wv * iv + 1.5).round_ties_even().clamp(0.0, 3.0);
                                let d = wv - (q - 1.5) * sc;
                                e += d * d;
                            }
                            ge = ge.min(e);
                        }
                        err += ge;
                    }
                    if err < best.0 {
                        best = (err, top);
                    }
                }
                tab = ladder_for(best.1, params_row);
            }
            let st = f16_to_f32(u16::from_le_bytes([params_row[2], params_row[3]]));
            let crow = &mut *codes_row;
            for g in 0..gpr {
                // Rung 0 is the exact zero; live groups start at 1.
                let nominal = if dead[g] {
                    0
                } else if st <= 0.0 {
                    1
                } else {
                    1 + ((lg[g] - lo_r) / st)
                        .round_ties_even()
                        .clamp(0.0, Q2TP_LMAX as f32) as usize
                };
                // The nominal rung comes from absmax/1.5, which pins the outer
                // level to the largest weight — that minimises the WORST error,
                // not the mean one, and with only four levels the difference is
                // large. Try the neighbouring rungs and keep whichever
                // reconstructs the group best. Costs encode time and nothing
                // else: the layout, the decoder and the kernels are unchanged.
                let tile = &row[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
                let c = if nominal == 0 || !search {
                    nominal
                } else {
                    let mut best = (f32::INFINITY, nominal);
                    for cand in nominal.saturating_sub(2).max(1)..=(nominal + 1).min(Q2TP_LMAX + 1)
                    {
                        let s = tab[cand];
                        if s <= 0.0 {
                            continue;
                        }
                        let inv = 1.0 / s;
                        let mut err = 0.0f32;
                        for &w in tile {
                            let q = (w * inv + 1.5).round_ties_even().clamp(0.0, 3.0);
                            let d = w - (q - 1.5) * s;
                            err += d * d;
                        }
                        if err < best.0 {
                            best = (err, cand);
                        }
                    }
                    best.1
                };
                q4tp_put_code(crow, g, c);
                let inv = if tab[c] > 0.0 { 1.0 / tab[c] } else { 0.0 };
                let dst = &mut chunks_row[g * Q2TP_CHUNK..(g + 1) * Q2TP_CHUNK];
                for (k, d) in dst.iter_mut().enumerate() {
                    let mut b = 0u8;
                    for j in 0..4 {
                        // (c − 1.5)·s: quantize w/s + 1.5 onto 0..=3.
                        let q = (tile[k * 4 + j] * inv + 1.5)
                            .round_ties_even()
                            .clamp(0.0, 3.0) as u8;
                        b |= q << (2 * j);
                    }
                    *d = b;
                }
            }
        },
    );
    chunks.extend_from_slice(&params);
    chunks.extend_from_slice(&codes);
    chunks
}

/// Multipliers `m` of `absmax/1.5` the placed ladder tries per group:
/// 0.300, 0.325, …, 1.050. The 4-level optimum of a real gate/up group sits
/// at 0.6–0.75 (MiniCPM5), out of reach of v1's −2..+1 rung window, which
/// spans only ~6 % in scale on a typical ladder.
const Q2TP_PLACE_MULTS: usize = 31;

#[inline]
fn q2tp_place_mult(k: usize) -> f32 {
    ((300 + 25 * k) as f64 / 1000.0) as f32
}

/// `x.round_ties_even().clamp(0.0, 3.0)` for the 2-bit grid, written so it
/// vectorizes on baseline x86-64 (no SSE4.1 round): after the clamp to
/// [−1, 4], adding and subtracting 1.5·2^23 rounds to the nearest integer
/// with ties to even (IEEE default rounding) — the same value as
/// `round_ties_even` (only a −0 comes back as +0, which no code or product
/// can tell apart). Rust never folds the pair away.
#[inline(always)]
pub(crate) fn q2_level(x: f32) -> f32 {
    const MAGIC: f32 = 12_582_912.0;
    let c = x.clamp(-1.0, 4.0);
    ((c + MAGIC) - MAGIC).clamp(0.0, 3.0)
}

/// Weighted squared error of one 32-group rounded onto the 2-bit grid at
/// scale `s` (`hw` = per-channel weights; all ones in weight space). Eight
/// fixed lanes reduced in a fixed order: it vectorizes and the sum does not
/// depend on the machine.
#[inline(always)]
pub(crate) fn q2tp_group_err(tile: &[f32; GROUP_SIZE], hw: &[f32; GROUP_SIZE], s: f32) -> f32 {
    let inv = 1.0 / s;
    let mut acc = [0f32; 8];
    for c in 0..GROUP_SIZE / 8 {
        for l in 0..8 {
            let w = tile[c * 8 + l];
            let d = w - (q2_level(w * inv + 1.5) - 1.5) * s;
            acc[l] += hw[c * 8 + l] * d * d;
        }
    }
    ((acc[0] + acc[1]) + (acc[2] + acc[3])) + ((acc[4] + acc[5]) + (acc[6] + acc[7]))
}

/// Best rung 1..=31 of one group by (weighted) round-to-nearest error over
/// the whole ladder; a tie keeps the lower rung.
#[inline]
pub(crate) fn q2tp_best_rung(
    tile: &[f32; GROUP_SIZE],
    hw: &[f32; GROUP_SIZE],
    tab: &[f32; 32],
) -> (f32, usize) {
    let mut best = (f32::INFINITY, 1usize);
    for (c, &s) in tab.iter().enumerate().skip(1) {
        if !(s > 0.0) {
            continue;
        }
        let e = q2tp_group_err(tile, hw, s);
        if e < best.0 {
            best = (e, c);
        }
    }
    best
}

/// Group `g` of a row and of its channel weights, as fixed-size arrays (the
/// error kernel's bounds checks then vanish).
#[inline(always)]
fn q2tp_group_pair<'a>(
    row: &'a [f32],
    hw: &'a [f32],
    g: usize,
) -> (&'a [f32; GROUP_SIZE], &'a [f32; GROUP_SIZE]) {
    let r = g * GROUP_SIZE..(g + 1) * GROUP_SIZE;
    (row[r.clone()].try_into().unwrap(), hw[r].try_into().unwrap())
}

/// Row params of a q2tp ladder with rung 1 at `lo_h` whose rung 31 provably
/// reaches `top` (step walked up by single ULPs, as in v1).
fn q2tp_row_params(lo_h: u16, top: f32) -> [u8; 4] {
    let lo_r = f16_to_f32(lo_h);
    let span = (top - lo_r).max(0.0);
    let mut st_h = f32_to_f16(span / Q2TP_LMAX as f32);
    for _ in 0..64 {
        let st = f16_to_f32(st_h);
        if st > 0.0 && lo_r + Q2TP_LMAX as f32 * st >= top {
            break;
        }
        st_h += 1;
    }
    let mut p = [0u8; 4];
    p[0..2].copy_from_slice(&lo_h.to_le_bytes());
    p[2..4].copy_from_slice(&st_h.to_le_bytes());
    p
}

/// The placed q2tp row ladder (v2 encoder, and the `placed` GPTQ rule).
///
/// Per live group the scale `m·absmax/1.5` (`m` over [`q2tp_place_mult`])
/// with the lowest `hw`-weighted rounding error; the row ladder spans those
/// scales, four ladder tops are tried as in v1, and the one whose FULL
/// 31-rung search reconstructs the row best is kept. Writes the row params
/// and each group's best rung (0 = an all-zero group). Zero bits: same
/// layout, reader and kernels — only where the scales sit changes.
pub(crate) fn q2tp_place_row(row: &[f32], hw: &[f32], params: &mut [u8], rungs: &mut [u8]) {
    debug_assert_eq!(row.len() % GROUP_SIZE, 0);
    debug_assert_eq!(row.len(), hw.len());
    let gpr = row.len() / GROUP_SIZE;
    let tile_at = |g: usize| q2tp_group_pair(row, hw, g);
    let mut lg = vec![0f32; gpr];
    let mut dead = vec![false; gpr];
    let (mut lo, mut hi) = (f32::INFINITY, f32::NEG_INFINITY);
    for g in 0..gpr {
        let (tile, h) = tile_at(g);
        let absmax = tile.iter().fold(0f32, |m, v| m.max(v.abs()));
        dead[g] = absmax == 0.0;
        let base = absmax / 1.5;
        let mut best = (f32::INFINITY, base);
        if !dead[g] {
            for k in 0..Q2TP_PLACE_MULTS {
                let s = base * q2tp_place_mult(k);
                if !(s > 0.0) {
                    continue;
                }
                let e = q2tp_group_err(tile, h, s);
                if e < best.0 {
                    best = (e, s);
                }
            }
        }
        lg[g] = det_log2(f16_scale(best.1));
        if !dead[g] {
            lo = lo.min(lg[g]);
            hi = hi.max(lg[g]);
        }
    }
    if !lo.is_finite() {
        lo = lg[0];
        hi = lo;
    }
    let lo_h = f32_to_f16(lo);
    let lo_r = f16_to_f32(lo_h);
    let mut best: Option<(f64, [u8; 4])> = None;
    let mut cand = vec![0u8; gpr];
    for frac in [1.0f32, 0.9, 0.8, 0.7] {
        let p = q2tp_row_params(lo_h, lo_r + (hi - lo_r) * frac);
        let tab = det_q2tp_ladder(&p, 0);
        let mut err = 0f64;
        for g in 0..gpr {
            cand[g] = if dead[g] {
                0
            } else {
                let (tile, h) = tile_at(g);
                let (e, c) = q2tp_best_rung(tile, h, &tab);
                err += e as f64;
                c as u8
            };
        }
        if best.as_ref().is_none_or(|b| err < b.0) {
            best = Some((err, p));
            rungs[..gpr].copy_from_slice(&cand);
        }
    }
    params[..4].copy_from_slice(&best.expect("four candidate ladders").1);
}

/// Round one row onto its ladder at the given rungs and pack the 2-bit plane
/// and the 5-bit rung codes (`codes_row` must start zeroed).
fn q2tp_pack_rtn(row: &[f32], params_row: &[u8], rungs: &[u8], chunks_row: &mut [u8], codes_row: &mut [u8]) {
    let tab = det_q2tp_ladder(params_row, 0);
    for (g, &c) in rungs.iter().enumerate() {
        let c = c as usize;
        q4tp_put_code(codes_row, g, c);
        let inv = if tab[c] > 0.0 { 1.0 / tab[c] } else { 0.0 };
        let tile = &row[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
        for (k, d) in chunks_row[g * Q2TP_CHUNK..(g + 1) * Q2TP_CHUNK].iter_mut().enumerate() {
            let mut b = 0u8;
            for j in 0..4 {
                b |= (q2_level(tile[k * 4 + j] * inv + 1.5) as u8) << (2 * j);
            }
            *d = b;
        }
    }
}

/// Multiply a whole q2tp row by `1/b`, `b = clip(num/den, 0.8, 1.25)` the
/// row's output regression slope. Every rung of a row is
/// `2^(lo + (c−1)·step)`, so that is exactly a shift of the f16 `lo` by
/// `−log2 b`: codes, rungs, step and bytes stay as they are.
///
/// Why: MSE-optimal 2-bit codes satisfy ⟨Ŵ, W − Ŵ⟩ ≈ 0, so every row comes
/// back SHRUNK by its error energy (~3.5 % at 2 bits on MiniCPM5 gate/up,
/// the same sign in all 84 tensors). Random error averages out through the
/// residual stream; a coherent shrink does not.
pub(crate) fn q2tp_unshrink_row(params_row: &mut [u8], num: f64, den: f64) {
    if !(den > 0.0) {
        return;
    }
    let b = num / den;
    if !b.is_finite() {
        return;
    }
    let b = b.clamp(0.8, 1.25);
    let lo = f16_to_f32(u16::from_le_bytes([params_row[0], params_row[1]]));
    let shifted = (lo as f64 - det_log2(b as f32) as f64) as f32;
    params_row[0..2].copy_from_slice(&f32_to_f16(shifted).to_le_bytes());
}

/// Weight-space debias of one encoded q2tp row: the slope
/// `b = ⟨W, Ŵ⟩ / ⟨W, W⟩` (no data — the identity metric) removed through `lo`.
fn q2tp_debias_row_weight_space(row: &[f32], chunks_row: &[u8], params_row: &mut [u8], codes_row: &[u8]) {
    let tab = det_q2tp_ladder(params_row, 0);
    let (mut num, mut den) = (0f64, 0f64);
    for (g, tile) in row.chunks_exact(GROUP_SIZE).enumerate() {
        let s = tab[cortiq_core::quant::q4tp_code(codes_row, g)];
        let chunk = &chunks_row[g * Q2TP_CHUNK..(g + 1) * Q2TP_CHUNK];
        for (k, &byte) in chunk.iter().enumerate() {
            for j in 0..4 {
                let w = tile[k * 4 + j] as f64;
                let wh = ((((byte >> (2 * j)) & 3) as f32 - 1.5) * s) as f64;
                num += w * wh;
                den += w * w;
            }
        }
    }
    q2tp_unshrink_row(params_row, num, den);
}

/// The v2 `q2tp` encoder (default): the zero-bit, data-free fix measured
/// on MiniCPM5-2B (q2tp profile, head/embed q8_2f: WikiText-2 ppl
/// 206.6 → 89.5). Three steps per row:
///
/// 1. place the ladder at the per-group MSE-optimal scale instead of
///    absmax/1.5 ([`q2tp_place_row`]) — weight MSE −40 %;
/// 2. pick each group's rung by a full 31-rung search and round to nearest;
/// 3. remove the output shrinkage those MSE-optimal codes cause, per row,
///    through the f16 `lo` ([`q2tp_unshrink_row`]) — worth −49 % ppl alone.
///
/// Same bytes per tensor, same reader, same kernels.
pub(crate) fn encode_q2tp_v2(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    debug_assert_eq!(vals.len(), out_dim * in_dim);
    debug_assert_eq!(in_dim % GROUP_SIZE, 0);
    let gpr = in_dim / GROUP_SIZE;
    let stride = q4tp_code_stride(gpr);
    let mut chunks = vec![0u8; out_dim * gpr * Q2TP_CHUNK];
    let mut params = vec![0u8; out_dim * 4];
    let mut codes = vec![0u8; out_dim * stride];
    let ones = vec![1.0f32; in_dim];
    encode_rows_parallel(
        out_dim,
        [gpr * Q2TP_CHUNK, 4, stride],
        [&mut chunks, &mut params, &mut codes],
        &|r, chunks_row, params_row, codes_row| {
            let row = &vals[r * in_dim..(r + 1) * in_dim];
            let mut rungs = vec![0u8; gpr];
            q2tp_place_row(row, &ones, params_row, &mut rungs);
            q2tp_pack_rtn(row, params_row, &rungs, chunks_row, codes_row);
            q2tp_debias_row_weight_space(row, chunks_row, params_row, codes_row);
        },
    );
    chunks.extend_from_slice(&params);
    chunks.extend_from_slice(&codes);
    chunks
}

/// `log2` for the ladder encoders in plain f64 IEEE arithmetic. The
/// platform libm (macOS vs glibc) disagrees in the last bit on some inputs,
/// which flips near-tie rung choices: the same source packed to different
/// q4tp bytes on aarch64 and x86_64. Basic IEEE operations are correctly
/// rounded everywhere and Rust never contracts them, so this is the same on
/// every machine. Accurate to well under an f32 ulp.
pub(crate) fn det_log2(x: f32) -> f32 {
    if x.is_nan() || x < 0.0 {
        return f32::NAN;
    }
    if x == 0.0 {
        return f32::NEG_INFINITY;
    }
    if x.is_infinite() {
        return f32::INFINITY;
    }
    // x = m · 2^e with m in [1, 2) (subnormals normalised through f64)
    let d = x as f64;
    let bits = d.to_bits();
    let e = ((bits >> 52) & 0x7ff) as i64 - 1023;
    let m = f64::from_bits((bits & ((1u64 << 52) - 1)) | (1023u64 << 52));
    // ln m = 2·atanh(t), t = (m − 1)/(m + 1) ∈ [0, 1/3)
    let t = (m - 1.0) / (m + 1.0);
    let t2 = t * t;
    let mut term = t;
    let mut sum = 0.0f64;
    let mut k = 1.0f64;
    for _ in 0..24 {
        sum += term / k;
        term *= t2;
        k += 2.0;
    }
    let ln_m = 2.0 * sum;
    (e as f64 + ln_m * std::f64::consts::LOG2_E) as f32
}

/// `exp2` for the ladder encoders, same reasoning as [`det_log2`].
pub(crate) fn det_exp2(y: f32) -> f32 {
    if y.is_nan() {
        return f32::NAN;
    }
    if y > 128.0 {
        return f32::INFINITY;
    }
    if y < -160.0 {
        return 0.0;
    }
    let yd = y as f64;
    let n = yd.floor();
    let f = yd - n; // [0, 1)
    // 2^f = e^(f·ln 2), Taylor to f64 precision (|z| < 0.7)
    let z = f * std::f64::consts::LN_2;
    let mut term = 1.0f64;
    let mut sum = 1.0f64;
    for k in 1..=24 {
        term *= z / k as f64;
        sum += term;
    }
    // exact scaling by 2^n (n in [-160, 128], within the f64 normal range)
    let scale = f64::from_bits(((n as i64 + 1023) as u64) << 52);
    (sum * scale) as f32
}

/// `q4tp_ladder` with [`det_exp2`]: the rung table the encoders choose codes
/// against (the runtime keeps the core ladder; the two differ by at most an
/// ulp, which only matters to the encoder's near-tie choices).
pub(crate) fn det_q4tp_ladder(params: &[u8], r: usize) -> [f32; 32] {
    let lo = f16_to_f32(u16::from_le_bytes([params[r * 4], params[r * 4 + 1]]));
    let st = f16_to_f32(u16::from_le_bytes([params[r * 4 + 2], params[r * 4 + 3]]));
    let ratio = det_exp2(st);
    let mut t = [0f32; 32];
    t[0] = det_exp2(lo);
    for c in 1..32 {
        t[c] = t[c - 1] * ratio;
    }
    t
}

/// `q2tp_ladder` over [`det_q4tp_ladder`] (rung 0 = the exact zero).
pub(crate) fn det_q2tp_ladder(params: &[u8], r: usize) -> [f32; 32] {
    let base = det_q4tp_ladder(params, r);
    let mut t = [0f32; 32];
    t[1..32].copy_from_slice(&base[..31]);
    t
}

pub(crate) fn encode_q4tp(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    debug_assert_eq!(vals.len(), out_dim * in_dim);
    debug_assert_eq!(in_dim % GROUP_SIZE, 0);
    let gpr = in_dim / GROUP_SIZE;
    let stride = q4tp_code_stride(gpr);
    let mut nib = vec![0u8; out_dim * gpr * Q4TP_NIB];
    let mut params = vec![0u8; out_dim * 4];
    let mut codes = vec![0u8; out_dim * stride];

    let search = q2tp_search();

    encode_rows_parallel(
        out_dim,
        [gpr * Q4TP_NIB, 4, stride],
        [&mut nib, &mut params, &mut codes],
        &|r, nib_row, params_row, codes_row| {
            let mut lg = vec![0f32; gpr];
            let mut dead = vec![false; gpr];

            let row = &vals[r * in_dim..(r + 1) * in_dim];
            let (mut lo, mut hi) = (f32::INFINITY, f32::NEG_INFINITY);
            for g in 0..gpr {
                let tile = &row[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
                let absmax = tile.iter().fold(0f32, |m, v| m.max(v.abs()));
                // An all-zero tile dequantizes to zero whatever its scale, so keep
                // it out of the row's range — otherwise `f16_scale`'s tiny floor
                // stretches the ladder and coarsens every live tile in the row.
                dead[g] = absmax == 0.0;
                lg[g] = det_log2(f16_scale(absmax / 7.0));
                if !dead[g] {
                    lo = lo.min(lg[g]);
                    hi = hi.max(lg[g]);
                }
            }
            if !lo.is_finite() {
                lo = lg[0];
                hi = lo;
            }

            let lo_h = f32_to_f16(lo);
            let lo_r = f16_to_f32(lo_h);
            let span = (hi - lo_r).max(0.0);
            let mut st_h = f32_to_f16(span / Q4TP_LMAX as f32);
            // Round-to-nearest on `step` can land just short of `hi`; walk up by
            // single ULPs until rung 31 provably covers the row's top scale.
            for _ in 0..64 {
                let st = f16_to_f32(st_h);
                if st > 0.0 && lo_r + Q4TP_LMAX as f32 * st >= hi {
                    break;
                }
                st_h += 1;
            }
            params_row[0..2].copy_from_slice(&lo_h.to_le_bytes());
            params_row[2..4].copy_from_slice(&st_h.to_le_bytes());

            let st = f16_to_f32(st_h);
            let tab = det_q4tp_ladder(params_row, 0);
            let crow = &mut *codes_row;
            for g in 0..gpr {
                let nominal = if dead[g] || st <= 0.0 {
                    0
                } else {
                    ((lg[g] - lo_r) / st)
                        .round_ties_even()
                        .clamp(0.0, Q4TP_LMAX as f32) as usize
                };
                let tile = &row[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
                // Same reasoning as q2tp: absmax fixes the outer level and one
                // outlier coarsens the group behind it. Sixteen levels forgive
                // more than four, so the win is smaller — but it is measured.
                let c = if dead[g] || st <= 0.0 || !search {
                    nominal
                } else {
                    let mut best = (f32::INFINITY, nominal);
                    for cand in nominal.saturating_sub(2)..=(nominal + 1).min(Q4TP_LMAX) {
                        let sc = tab[cand];
                        if sc <= 0.0 {
                            continue;
                        }
                        let iv = 1.0 / sc;
                        let mut err = 0.0f32;
                        for &wv in tile {
                            let q = (wv * iv).round_ties_even().clamp(-8.0, 7.0);
                            let d = wv - q * sc;
                            err += d * d;
                        }
                        if err < best.0 {
                            best = (err, cand);
                        }
                    }
                    best.1
                };
                q4tp_put_code(crow, g, c);
                let inv = if tab[c] > 0.0 { 1.0 / tab[c] } else { 0.0 };
                let dst = &mut nib_row[g * Q4TP_NIB..(g + 1) * Q4TP_NIB];
                for k in 0..16 {
                    let q0 =
                        ((tile[k * 2] * inv).round_ties_even().clamp(-8.0, 7.0) as i8 + 8) as u8;
                    let q1 = ((tile[k * 2 + 1] * inv).round_ties_even().clamp(-8.0, 7.0) as i8 + 8)
                        as u8;
                    dst[k] = (q0 & 0x0F) | (q1 << 4);
                }
            }
        },
    );

    nib.extend_from_slice(&params);
    nib.extend_from_slice(&codes);
    nib
}

/// Re-encode f32 values into an EXISTING tensor's dtype — the offline passes
/// (AWNP) rewrite weights and must put them back in the layout the file
/// already uses, not pick a new one.
pub(crate) fn encode_for(dtype: TensorDtype, vals: &[f32], rows: usize, cols: usize) -> Vec<u8> {
    match dtype {
        TensorDtype::Q4Tiled => encode_q4_tiled(vals, rows, cols),
        TensorDtype::Q4TiledP => encode_q4tp(vals, rows, cols),
        TensorDtype::Q4Block => encode_q4_block(vals),
        TensorDtype::Q8Row => encode_q8_row(vals, rows, cols),
        TensorDtype::Q8_2f => encode_q8_2f(vals, rows, cols),
        _ => encode_f16(vals),
    }
}

/// q1 (dtype 12): per 32-group tile `[f16 scale][4B sign bits]`,
/// bit k of byte j (LSB-first) = weight j·8+k; value = s·(2·bit−1).
/// Scale = group mean |v| — the L2-optimal binary level; for a
/// 1-bit-TRAINED model whose group weights already sit on ±s this
/// recovers the level exactly (encoding is lossless up to f16 range).
fn encode_q1(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    debug_assert_eq!(vals.len(), out_dim * in_dim);
    debug_assert_eq!(in_dim % GROUP_SIZE, 0);
    let n_groups = vals.len() / GROUP_SIZE;
    let mut out = Vec::with_capacity(n_groups * 6);
    for g in 0..n_groups {
        let grp = &vals[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
        let mean_abs = grp.iter().map(|v| v.abs()).sum::<f32>() / GROUP_SIZE as f32;
        let s = f16_scale(mean_abs);
        out.extend_from_slice(&f32_to_f16(s).to_le_bytes());
        for j in 0..GROUP_SIZE / 8 {
            let mut byte = 0u8;
            for k in 0..8 {
                if grp[j * 8 + k] >= 0.0 {
                    byte |= 1 << k;
                }
            }
            out.push(byte);
        }
    }
    out
}

/// Error-diffusion ("перетекание") q1 encoder — the training-free PTQ path
/// for a NON-1-bit-trained model. Naïve q1 throws away every weight's
/// magnitude, keeping only its sign against a shared group scale; for a
/// normal checkpoint that is catastrophic. Here the per-weight rounding
/// residual `w − ŵ` is carried FORWARD along the row's input dimension and
/// folded into the next sign decision (`sign(w + carry)`), so the row's
/// running sum — hence its contribution to the dot product for the
/// slowly-varying part of the activation — is preserved rather than
/// discarded. Same on-disk `Q1` tile as `encode_q1` (reuses the kernel and
/// GPU path unchanged), and bit-identical to it on a genuinely 1-bit model
/// (near-constant |w| per group ⇒ `carry ≈ 0` ⇒ the sign never flips).
/// The carry resets at each row start (each output is an independent sum).
fn encode_q1_ef(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    debug_assert_eq!(vals.len(), out_dim * in_dim);
    debug_assert_eq!(in_dim % GROUP_SIZE, 0);
    let groups_per_row = in_dim / GROUP_SIZE;
    let n_groups = vals.len() / GROUP_SIZE;
    let mut out = Vec::with_capacity(n_groups * 6);
    let mut carry = 0.0f32;
    for g in 0..n_groups {
        if g % groups_per_row == 0 {
            carry = 0.0; // new output row: its dot product starts fresh
        }
        let grp = &vals[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
        let mean_abs = grp.iter().map(|v| v.abs()).sum::<f32>() / GROUP_SIZE as f32;
        let s = f16_scale(mean_abs);
        out.extend_from_slice(&f32_to_f16(s).to_le_bytes());
        for j in 0..GROUP_SIZE / 8 {
            let mut byte = 0u8;
            for k in 0..8 {
                let w = grp[j * 8 + k];
                let v = w + carry;
                let bit = v >= 0.0;
                if bit {
                    byte |= 1 << k;
                }
                carry = v - if bit { s } else { -s };
            }
            out.push(byte);
        }
    }
    out
}

/// Fraction of weights the `q1s` mask keeps at full precision (the outlier
/// budget). `CMF_Q1S_KEEP` overrides; default 1%. Clamped to [0, 25%].
fn q1s_keep_frac() -> f32 {
    std::env::var("CMF_Q1S_KEEP")
        .ok()
        .and_then(|v| v.parse::<f32>().ok())
        .unwrap_or(0.01)
        .clamp(0.0, 0.25)
}

/// 1-bit PTQ with an outlier mask (Stage 2a of the error-feedback
/// path). Keeps the top `keep_frac` of weights by |value| — the heavy tail
/// a normal checkpoint carries — at full f16 precision in a sparse overlay,
/// and binarizes the rest with the error-diffusion base, EXCLUDING the
/// outliers from each group's shared ±s scale (an outlier must not inflate
/// the level the bulk is quantized against). This is the |W| field of the
/// two-field mask; the activation field (`|W|·RMS(x)`) and the covariance
/// fold `Σ_PS·Σ_SS⁻¹` come from the calibration path on top of this.
fn encode_q1s(vals: &[f32], out_dim: usize, in_dim: usize, keep_frac: f32) -> Vec<u8> {
    debug_assert_eq!(vals.len(), out_dim * in_dim);
    debug_assert_eq!(in_dim % GROUP_SIZE, 0);
    let n = vals.len();
    let n_out = (((n as f32) * keep_frac).round() as usize).min(n);
    // Outlier threshold via nth_element (O(n)): the (n − n_out)-th smallest
    // |w| — weights at or above it are the kept heavy tail.
    let threshold = if n_out == 0 {
        f32::INFINITY
    } else {
        let mut absv: Vec<f32> = vals.iter().map(|v| v.abs()).collect();
        let k = n - n_out;
        absv.select_nth_unstable_by(k, |a, b| a.partial_cmp(b).unwrap());
        absv[k]
    };
    let is_out: Vec<bool> = (0..n)
        .map(|i| n_out > 0 && vals[i].abs() >= threshold)
        .collect();

    let groups_per_row = in_dim / GROUP_SIZE;
    let n_groups = n / GROUP_SIZE;
    let n_out_actual = is_out.iter().filter(|&&o| o).count();
    let mut out = Vec::with_capacity(n_groups * 6 + 4 + n_out_actual * 6);
    let mut carry = 0.0f32;
    for g in 0..n_groups {
        if g % groups_per_row == 0 {
            carry = 0.0;
        }
        let base = g * GROUP_SIZE;
        // Scale = mean |w| over the NON-outlier weights of the group.
        let mut sum = 0.0f32;
        let mut cnt = 0usize;
        for j in 0..GROUP_SIZE {
            if !is_out[base + j] {
                sum += vals[base + j].abs();
                cnt += 1;
            }
        }
        let s = f16_scale(if cnt > 0 { sum / cnt as f32 } else { 0.0 });
        out.extend_from_slice(&f32_to_f16(s).to_le_bytes());
        for jb in 0..GROUP_SIZE / 8 {
            let mut byte = 0u8;
            for k in 0..8 {
                let i = base + jb * 8 + k;
                if is_out[i] {
                    // Outlier: bit is only a sign hint (the overlay restores
                    // the exact value); it carries no error forward.
                    if vals[i] >= 0.0 {
                        byte |= 1 << k;
                    }
                } else {
                    let v = vals[i] + carry;
                    let bit = v >= 0.0;
                    if bit {
                        byte |= 1 << k;
                    }
                    carry = v - if bit { s } else { -s };
                }
            }
            out.push(byte);
        }
    }
    // Sparse outlier section: [u32 count][count × (u32 index, f16 value)].
    out.extend_from_slice(&(n_out_actual as u32).to_le_bytes());
    for (i, &o) in is_out.iter().enumerate() {
        if o {
            out.extend_from_slice(&(i as u32).to_le_bytes());
            out.extend_from_slice(&f32_to_f16(vals[i]).to_le_bytes());
        }
    }
    out
}

/// Exact ternary re-encode for weights that ALREADY sit on `{−s, 0, +s}`
/// within every 32-group — a ternary-trained checkpoint such as the STQ1_0
/// GGUF of Hy-MT2-1.8B (AngelSlim 1.25-bit). Per group: scale = the
/// group's max |w| (an f16 in the source, so it survives the round trip),
/// base-3 codes, and an EMPTY outlier overlay. `dequant_q1t` reproduces
/// the input bit for bit; no calibration, no requantization.
pub(crate) fn encode_q1t_exact(
    vals: &[f32],
    out_dim: usize,
    in_dim: usize,
) -> anyhow::Result<Vec<u8>> {
    use cortiq_core::quant::{GROUP_SIZE, Q1T_TILE, q1t_pack};
    anyhow::ensure!(
        vals.len() == out_dim * in_dim,
        "q1t exact: {} values != {out_dim}×{in_dim}",
        vals.len()
    );
    anyhow::ensure!(
        in_dim % GROUP_SIZE == 0,
        "q1t exact: in_dim {in_dim} is not a multiple of {GROUP_SIZE}"
    );
    anyhow::ensure!(
        in_dim <= u16::MAX as usize + 1,
        "q1t overlay: in_dim {in_dim} exceeds u16"
    );
    let n_groups = vals.len() / GROUP_SIZE;
    let mut out = Vec::with_capacity(n_groups * Q1T_TILE + (out_dim + 1) * 4);
    for g in 0..n_groups {
        let grp = &vals[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
        let s = grp.iter().fold(0f32, |m, v| m.max(v.abs()));
        let s16 = f32_to_f16(s);
        let s_rt = f16_to_f32(s16);
        let mut codes = [0u8; 7];
        for (k, &v) in grp.iter().enumerate() {
            let code = if v == 0.0 {
                0
            } else {
                anyhow::ensure!(
                    (v.abs() - s_rt).abs() <= s_rt * 1e-3,
                    "q1t exact: group {g} is not ternary (|w| {} vs scale {s_rt})",
                    v.abs()
                );
                if v > 0.0 { 1 } else { 2 }
            };
            q1t_pack(&mut codes, k, code);
        }
        out.extend_from_slice(&s16.to_le_bytes());
        out.extend_from_slice(&codes);
    }
    // Empty per-row overlay: row_ptr[out_dim + 1], all zero.
    out.resize(out.len() + (out_dim + 1) * 4, 0);
    Ok(out)
}

pub(crate) fn encode_q8_2f(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    // Column field: RMS over rows, f16-rounded (the decoder multiplies by these).
    // Both halves run across the encode threads: this is the codec a 20 B DiT
    // gets packed in, and serially it was the slowest encoder in the crate by
    // an order of magnitude while q4tp next to it used the whole machine.
    // The column reduction is blocked at a FIXED width, not at one block per
    // thread: f64 addition does not associate, so a thread-shaped grouping
    // would make the bytes depend on the core count. Blocks are summed in
    // index order, which is the same arithmetic on one thread and on thirty.
    const CBLK: usize = 256;
    let nblk = out_dim.div_ceil(CBLK);
    let threads = encode_threads().min(nblk.max(1)).max(1);
    let mut col = vec![0f32; in_dim];
    {
        let mut blocks: Vec<Vec<f64>> = vec![Vec::new(); nblk];
        let per = nblk.div_ceil(threads);
        std::thread::scope(|sc| {
            for (ti, chunk) in blocks.chunks_mut(per).enumerate() {
                sc.spawn(move || {
                    for (bi, slot) in chunk.iter_mut().enumerate() {
                        let b = ti * per + bi;
                        let (r0, r1) = (b * CBLK, ((b + 1) * CBLK).min(out_dim));
                        let mut acc = vec![0f64; in_dim];
                        for o in r0..r1 {
                            for (i, a) in acc.iter_mut().enumerate() {
                                let v = vals[o * in_dim + i] as f64;
                                *a += v * v;
                            }
                        }
                        *slot = acc;
                    }
                });
            }
        });
        for (i, c) in col.iter_mut().enumerate() {
            let mut acc = 0f64;
            for b in &blocks {
                acc += b[i];
            }
            let rms = (acc / out_dim as f64).sqrt().max(1e-12) as f32;
            *c = f16_to_f32(f32_to_f16(rms)).max(F16_TINY);
        }
    }
    let mut q = vec![0u8; out_dim * in_dim];
    let mut scales = vec![0u8; out_dim * 2];
    let mut unused = vec![0u8; out_dim];
    encode_rows_parallel(
        out_dim,
        [in_dim, 2, 1],
        [&mut q, &mut scales, &mut unused],
        &|o: usize, qrow: &mut [u8], srow: &mut [u8], _: &mut [u8]| {
            let src = &vals[o * in_dim..(o + 1) * in_dim];
            let mut absmax = 0f32;
            for (v, c) in src.iter().zip(&col) {
                absmax = absmax.max((v / c).abs());
            }
            let scale = f16_scale(absmax.max(1e-12) / 127.0);
            for ((d, v), c) in qrow.iter_mut().zip(src).zip(&col) {
                *d = ((v / c) / scale).round_ties_even().clamp(-127.0, 127.0) as i8 as u8;
            }
            srow.copy_from_slice(&f32_to_f16(scale).to_le_bytes());
        },
    );
    let mut out = q;
    out.extend_from_slice(&scales);
    for &c in &col {
        out.extend_from_slice(&f32_to_f16(c).to_le_bytes());
    }
    out
}

// Grouped variable-bit (v-bit) encoder — the weight-only (round-to-nearest) path
// of the reference converter. On-disk layout read by `cortiq_core::dequant_vbit`:
//   [u8 bits: rows][f16 scales: rows·(in/32)][per row: ceil(in·b/8) bytes,
//    MSB-first b-bit codes, zero-padded]. w = (u − L)·scale, L = 2^(b−1)−1.
// The GPTQ / calibrated variant (needs a Hessian) stays in the Python converter.
const VBIT_LEVELS: [u8; 5] = [3, 4, 5, 6, 8];
/// Target mean bit-width for VBIT water-filling. Default 4.25; overridable via
/// `cortiq convert --mean-bits` (stored ×1000 in a static to avoid signature churn).
static VBIT_MEAN_BITS_MILLI: AtomicU32 = AtomicU32::new(4250);
/// Set the VBIT target mean bit-width (converter CLI knob). Clamped to [3.0, 8.0].
pub fn set_vbit_mean_bits(bits: f32) {
    VBIT_MEAN_BITS_MILLI.store((bits.clamp(3.0, 8.0) * 1000.0) as u32, Ordering::Relaxed);
}
fn vbit_mean_bits() -> f32 {
    VBIT_MEAN_BITS_MILLI.load(Ordering::Relaxed) as f32 / 1000.0
}

/// Snap `x` to the nearest allowed bit-width (first wins on a tie, like argmin).
fn vbit_snap_level(x: f32) -> u8 {
    let mut best = VBIT_LEVELS[0];
    let mut bestd = (x - best as f32).abs();
    for &lv in &VBIT_LEVELS[1..] {
        let d = (x - lv as f32).abs();
        if d < bestd {
            bestd = d;
            best = lv;
        }
    }
    best
}

/// Per-row bit-width via water-filling over log2 row amplitude (floor 3 bits).
fn vbit_bits(vals: &[f32], out_dim: usize, in_dim: usize, mean_bits: f32) -> Vec<u8> {
    let a: Vec<f32> = (0..out_dim)
        .map(|o| {
            let mx = vals[o * in_dim..(o + 1) * in_dim]
                .iter()
                .fold(0f32, |m, v| m.max(v.abs()));
            mx.max(1e-12).log2()
        })
        .collect();
    let amean = a.iter().sum::<f32>() / out_dim as f32;
    a.iter()
        .map(|&ar| vbit_snap_level(mean_bits + (ar - amean)).max(3))
        .collect()
}

/// Big-endian (MSB-first) bit packer; the last byte of each row is zero-padded.
struct BitWriter {
    buf: Vec<u8>,
    cur: u8,
    nbits: u8,
}
impl BitWriter {
    fn with_capacity(n: usize) -> Self {
        Self {
            buf: Vec::with_capacity(n),
            cur: 0,
            nbits: 0,
        }
    }
    fn push(&mut self, v: u32, b: u32) {
        for i in (0..b).rev() {
            self.cur = (self.cur << 1) | ((v >> i) & 1) as u8;
            self.nbits += 1;
            if self.nbits == 8 {
                self.buf.push(self.cur);
                self.cur = 0;
                self.nbits = 0;
            }
        }
    }
    fn flush_row(&mut self) {
        if self.nbits > 0 {
            self.buf.push(self.cur << (8 - self.nbits));
            self.cur = 0;
            self.nbits = 0;
        }
    }
}

fn encode_vbit(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    let ng = in_dim / GROUP_SIZE;
    let bits = vbit_bits(vals, out_dim, in_dim, vbit_mean_bits());

    // Per-(row, group) scale = group absmax / L, f16-rounded and floored.
    let mut scale = vec![0f32; out_dim * ng];
    let mut sc_bytes = Vec::with_capacity(out_dim * ng * 2);
    for o in 0..out_dim {
        let l = (2f32.powi(bits[o] as i32 - 1) - 1.0).max(1.0);
        for g in 0..ng {
            let base = o * in_dim + g * GROUP_SIZE;
            let mx = vals[base..base + GROUP_SIZE]
                .iter()
                .fold(0f32, |m, v| m.max(v.abs()));
            let s = f16_scale(mx / l);
            scale[o * ng + g] = s;
            sc_bytes.extend_from_slice(&f32_to_f16(s).to_le_bytes());
        }
    }

    let mut out = Vec::with_capacity(out_dim + sc_bytes.len() + out_dim * in_dim);
    out.extend_from_slice(&bits);
    out.extend_from_slice(&sc_bytes);
    let mut bw = BitWriter::with_capacity(out_dim * in_dim);
    for o in 0..out_dim {
        let b = bits[o] as u32;
        let l = 2f32.powi(bits[o] as i32 - 1) - 1.0;
        let maxq = 2f32.powi(bits[o] as i32) - 1.0;
        for c in 0..in_dim {
            let s = scale[o * ng + c / GROUP_SIZE];
            let q = ((vals[o * in_dim + c] / s).round_ties_even() + l).clamp(0.0, maxq) as u32;
            bw.push(q, b);
        }
        bw.flush_row();
    }
    out.extend_from_slice(&bw.buf);
    out
}

/// `vbit_ro` (§4.2): the same bits/scales/packed encoding as
/// `encode_vbit`, plus `u32 row_offsets[rows+1]` (relative to the
/// packed area) between the scales and the packed rows — readers get
/// O(1) row access without a prefix scan. New dtype id; the byte
/// semantics of legacy `vbit` are untouched.
fn encode_vbit_ro(vals: &[f32], out_dim: usize, in_dim: usize) -> Vec<u8> {
    let legacy = encode_vbit(vals, out_dim, in_dim);
    let ng = in_dim / GROUP_SIZE;
    let sc_len = out_dim * ng * 2;
    let (head, packed) = legacy.split_at(out_dim + sc_len);
    let bits = &head[..out_dim];
    let mut out = Vec::with_capacity(legacy.len() + (out_dim + 1) * 4);
    out.extend_from_slice(head);
    let mut off = 0u32;
    for &b in bits {
        out.extend_from_slice(&off.to_le_bytes());
        off += ((in_dim * b as usize).div_ceil(8)) as u32;
    }
    out.extend_from_slice(&off.to_le_bytes());
    debug_assert_eq!(off as usize, packed.len());
    out.extend_from_slice(packed);
    out
}

/// f16 blob for a 1-D / small tensor.
pub(crate) fn encode_f16(vals: &[f32]) -> Vec<u8> {
    let mut out = Vec::with_capacity(vals.len() * 2);
    for &v in vals {
        out.extend_from_slice(&f32_to_f16(v).to_le_bytes());
    }
    out
}

/// Decode a safetensors dtype blob into f32 values.
/// Move every produced payload out of RAM and straight into the output file,
/// then drop it. Called after each source shard, so peak residency is one
/// shard's tensors rather than the model's — and because the writer streams
/// into the final file, peak DISK is the finished model rather than twice it.
const RAW_AUX_COPY_CHUNK: usize = 8 * 1024 * 1024;

/// Copy a native auxiliary tensor without decoding it or putting it through a
/// quantizer. DeepSeek-V4.1's Engram embedding planes are enormous FP8 byte
/// tables; they remain `U8` records with their original 2-D shape and are
/// interpreted by the V4.1 runtime. The writer hashes and copies them in
/// bounded chunks directly from the safetensors mmap.
fn push_raw_u8_tensor(
    writers: &mut [Option<cortiq_core::format::CmfStreamWriter>],
    active: &[bool],
    name: &str,
    shape: &[usize],
    data: &[u8],
) -> anyhow::Result<()> {
    anyhow::ensure!(
        writers.len() == active.len(),
        "raw tensor output count mismatch"
    );
    let expected = shape.iter().try_fold(1usize, |n, &d| n.checked_mul(d));
    let expected =
        expected.ok_or_else(|| anyhow::anyhow!("raw tensor '{name}': shape overflow"))?;
    anyhow::ensure!(
        expected == data.len(),
        "raw tensor '{name}': {} bytes for shape {:?} (expected {})",
        data.len(),
        shape,
        expected
    );
    for (i, (writer, &is_active)) in writers.iter_mut().zip(active).enumerate() {
        if !is_active {
            continue;
        }
        let writer = writer
            .as_mut()
            .ok_or_else(|| anyhow::anyhow!("output {i} has no active writer for '{name}'"))?;
        writer
            .push_bounded(name, TensorDtype::U8, shape, data, RAW_AUX_COPY_CHUNK)
            .map_err(|e| anyhow::anyhow!("write raw tensor '{name}': {e}"))?;
    }
    Ok(())
}

fn drain_to_writer(
    tensors: &mut Vec<TensorSpec>,
    writer: &mut cortiq_core::format::CmfStreamWriter,
    exclusion: &TensorExclusion,
) -> anyhow::Result<()> {
    for t in tensors.drain(..) {
        if exclusion.skip_output(&t.name) {
            continue;
        }
        writer
            .push(&t.name, t.dtype, &t.shape, &t.data)
            .map_err(|e| anyhow::anyhow!("write tensor '{}': {e}", t.name))?;
    }
    Ok(())
}

pub(crate) fn to_f32(dtype: &str, raw: &[u8]) -> anyhow::Result<Vec<f32>> {
    Ok(match dtype {
        "F32" => raw
            .chunks_exact(4)
            .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
            .collect(),
        "F16" => raw
            .chunks_exact(2)
            .map(|b| f16_to_f32(u16::from_le_bytes([b[0], b[1]])))
            .collect(),
        "BF16" => raw
            .chunks_exact(2)
            .map(|b| bf16_to_f32(u16::from_le_bytes([b[0], b[1]])))
            .collect(),
        // DeepSeek-V4's hash-routing table is I64 token→expert indices, and
        // I32 shows up in the same role elsewhere. They are LOOKUP TABLES,
        // not weights: widen to f32 so the pipeline can carry them, and let
        // the writer's force_f16 rule keep them exact.
        "I64" => raw
            .chunks_exact(8)
            .map(|b| i64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]) as f32)
            .collect(),
        "I32" => raw
            .chunks_exact(4)
            .map(|b| i32::from_le_bytes([b[0], b[1], b[2], b[3]]) as f32)
            .collect(),
        other => anyhow::bail!("unsupported safetensors dtype '{other}' (need F32/F16/BF16)"),
    })
}

/// Reorder the source MLX Conv3d patch kernel into the flattened row layout
/// consumed by the CMF Qwen3 vision tower.  MLX's kernel is
/// `[out, temporal, patch_h, patch_w, in_channels]`; the CMF preprocessor
/// presents each patch as `[in_channels, temporal, patch_h, patch_w]`.
fn flatten_prism_patch_weight(
    dtype: &str,
    shape: &[usize],
    raw: &[u8],
) -> anyhow::Result<(Vec<usize>, Vec<f32>)> {
    anyhow::ensure!(
        dtype == "F16",
        "Prism vision patch kernel must be F16, got {dtype}"
    );
    anyhow::ensure!(
        shape.len() == 5,
        "Prism vision patch kernel must be [out,t,h,w,c], got {shape:?}"
    );
    let [out, temporal, patch_h, patch_w, channels] =
        [shape[0], shape[1], shape[2], shape[3], shape[4]];
    anyhow::ensure!(
        [out, temporal, patch_h, patch_w, channels]
            .iter()
            .all(|&n| n > 0),
        "Prism vision patch kernel has an empty dimension: {shape:?}"
    );
    let source = to_f32(dtype, raw)?;
    let source_len = out
        .checked_mul(temporal)
        .and_then(|n| n.checked_mul(patch_h))
        .and_then(|n| n.checked_mul(patch_w))
        .and_then(|n| n.checked_mul(channels))
        .ok_or_else(|| anyhow::anyhow!("Prism vision patch kernel shape overflows"))?;
    anyhow::ensure!(
        source.len() == source_len,
        "Prism vision patch kernel has {} values, expected {source_len}",
        source.len()
    );
    let patch_width = channels
        .checked_mul(temporal)
        .and_then(|n| n.checked_mul(patch_h))
        .and_then(|n| n.checked_mul(patch_w))
        .ok_or_else(|| anyhow::anyhow!("Prism vision patch width overflows"))?;
    let mut out_rows = vec![0.0f32; out * patch_width];
    for o in 0..out {
        for c in 0..channels {
            for t in 0..temporal {
                for y in 0..patch_h {
                    for x in 0..patch_w {
                        let src =
                            ((((o * temporal + t) * patch_h + y) * patch_w + x) * channels) + c;
                        let dst =
                            ((((o * channels + c) * temporal + t) * patch_h + y) * patch_w) + x;
                        out_rows[dst] = source[src];
                    }
                }
            }
        }
    }
    Ok((vec![out, patch_width], out_rows))
}

/// OCP MXFP4 (compressed-tensors "mxfp4-pack-quantized", Kimi-K3):
/// two FP4-E2M1 values per byte (LOW nibble = even element), groups of
/// 32 along the last axis, one E8M0 scale byte (2^(k−127)) per group.
pub(crate) fn unpack_mxfp4(
    packed: &[u8],
    scales: &[u8],
    rows: usize,
    cols_packed: usize,
) -> anyhow::Result<Vec<f32>> {
    // E2M1, bias 1: e=0 subnormal m·0.5; e≥1 normal (1+m/2)·2^(e−1).
    const LUT: [f32; 8] = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0];
    let cols = cols_packed * 2;
    anyhow::ensure!(cols % 32 == 0, "mxfp4: cols {cols} not a multiple of 32");
    let gpr = cols / 32;
    anyhow::ensure!(
        packed.len() == rows * cols_packed && scales.len() == rows * gpr,
        "mxfp4: packed {} / scales {} vs rows {rows} cols {cols}",
        packed.len(),
        scales.len()
    );
    let mut out = vec![0.0f32; rows * cols];
    if rows == 0 || cols == 0 {
        return Ok(out);
    }
    // Rows are independent, so the decode splits on rows exactly like the
    // FP8 decoder: a 256-expert layer is 768 of these matrices, and the
    // serial loop was a visible share of a conversion's wall time.
    let threads = encode_threads().min(rows).max(1);
    let rows_per_thread = rows.div_ceil(threads);
    std::thread::scope(|scope| -> anyhow::Result<()> {
        let mut workers = Vec::with_capacity(threads);
        for (chunk_index, out_chunk) in out.chunks_mut(rows_per_thread * cols).enumerate() {
            let first_row = chunk_index * rows_per_thread;
            workers.push(scope.spawn(move || -> anyhow::Result<()> {
                for (local_row, out_row) in out_chunk.chunks_mut(cols).enumerate() {
                    let r = first_row + local_row;
                    for g in 0..gpr {
                        let k = scales[r * gpr + g];
                        // E8M0: value = 2^(k−127); 255 = NaN per OCP — refuse loudly.
                        anyhow::ensure!(k != 255, "mxfp4: NaN scale at row {r} group {g}");
                        let scale = (k as f32 - 127.0).exp2();
                        for b in 0..16 {
                            let byte = packed[r * cols_packed + g * 16 + b];
                            for (half, nib) in [(0usize, byte & 0x0F), (1usize, byte >> 4)] {
                                let mag = LUT[(nib & 0x7) as usize];
                                let v = if nib & 0x8 != 0 { -mag } else { mag };
                                out_row[g * 32 + b * 2 + half] = v * scale;
                            }
                        }
                    }
                }
                Ok(())
            }));
        }
        for worker in workers {
            worker
                .join()
                .map_err(|_| anyhow::anyhow!("mxfp4 worker panicked"))??;
        }
        Ok(())
    })?;
    Ok(out)
}

/// NVIDIA ModelOpt NVFP4 (`hf_quant_config.json` / `quantization_config`
/// `quant_algo: NVFP4`): `weight` U8 `[rows, cols/2]` E2M1 pairs (LOW nibble
/// = even element, as MXFP4), `weight_scale` F8_E4M3 `[rows, cols/16]` (one
/// per 16 elements, linear row-major on disk — not the swizzled runtime
/// layout), `weight_scale_2` F32 scalar (the per-tensor decode multiplier,
/// `amax / (6·448)`). Decoded as vLLM's `dequantize_to_dtype` does:
/// `w = e2m1 · (f32(e4m3_scale) · weight_scale_2)`, the two scales multiplied
/// first. A NaN block scale (E4M3 `0x7F`/`0xFF`) or a non-finite global
/// scale is refused.
pub(crate) fn unpack_nvfp4(
    packed: &[u8],
    scales_e4m3: &[u8],
    global_scale: f32,
    rows: usize,
    cols_packed: usize,
) -> anyhow::Result<Vec<f32>> {
    const BLOCK: usize = 16;
    let cols = cols_packed * 2;
    anyhow::ensure!(cols % BLOCK == 0, "nvfp4: cols {cols} not a multiple of {BLOCK}");
    anyhow::ensure!(
        global_scale.is_finite(),
        "nvfp4: non-finite weight_scale_2 {global_scale}"
    );
    let bpr = cols / BLOCK;
    anyhow::ensure!(
        packed.len() == rows * cols_packed && scales_e4m3.len() == rows * bpr,
        "nvfp4: packed {} / scales {} vs rows {rows} cols {cols}",
        packed.len(),
        scales_e4m3.len()
    );
    let lut = &cortiq_core::mxfp4::E2M1_LUT;
    let mut out = vec![0.0f32; rows * cols];
    if rows == 0 {
        return Ok(out);
    }
    let threads = encode_threads().min(rows).max(1);
    let rows_per_thread = rows.div_ceil(threads);
    std::thread::scope(|scope| -> anyhow::Result<()> {
        let mut workers = Vec::with_capacity(threads);
        for (chunk_index, out_chunk) in out.chunks_mut(rows_per_thread * cols).enumerate() {
            let first_row = chunk_index * rows_per_thread;
            workers.push(scope.spawn(move || -> anyhow::Result<()> {
                for (local_row, out_row) in out_chunk.chunks_mut(cols).enumerate() {
                    let r = first_row + local_row;
                    for b in 0..bpr {
                        let sf = fp8_e4m3_to_f32(scales_e4m3[r * bpr + b]);
                        anyhow::ensure!(sf.is_finite(), "nvfp4: NaN block scale at row {r} block {b}");
                        let s = sf * global_scale;
                        for j in 0..BLOCK / 2 {
                            let byte = packed[r * cols_packed + b * (BLOCK / 2) + j];
                            out_row[b * BLOCK + 2 * j] = lut[(byte & 0x0F) as usize] * s;
                            out_row[b * BLOCK + 2 * j + 1] = lut[(byte >> 4) as usize] * s;
                        }
                    }
                }
                Ok(())
            }));
        }
        for worker in workers {
            worker
                .join()
                .map_err(|_| anyhow::anyhow!("nvfp4 worker panicked"))??;
        }
        Ok(())
    })?;
    Ok(out)
}

/// The NVFP4 group size when the checkpoint declares ModelOpt NVFP4, either
/// in `hf_quant_config.json` (`{"quantization": {"quant_algo": "NVFP4",
/// "group_size": 16}}`) or in `config.json`'s `quantization_config`
/// (`quant_algo` NVFP4 / W4A16_NVFP4, `quant_method` modelopt). None for
/// every other checkpoint. compressed-tensors' `nvfp4-pack-quantized`
/// (packed `weight_packed`, an INVERSE `weight_global_scale`) is a different
/// layout and is refused rather than misread as MXFP4.
pub(crate) fn nvfp4_source(dir: &Path, config: &serde_json::Value) -> anyhow::Result<Option<usize>> {
    let algo_is_nvfp4 = |q: &serde_json::Value| {
        q.get("quant_algo")
            .and_then(|v| v.as_str())
            .is_some_and(|a| a.to_ascii_uppercase().contains("NVFP4"))
    };
    let group_of = |q: &serde_json::Value| -> anyhow::Result<usize> {
        let g = q
            .get("group_size")
            .and_then(|v| v.as_u64())
            .or_else(|| {
                q.get("config_groups")?
                    .as_object()?
                    .values()
                    .find_map(|g| g.get("weights")?.get("group_size")?.as_u64())
            })
            .unwrap_or(16) as usize;
        anyhow::ensure!(g == 16, "NVFP4 checkpoint declares group_size {g}; only 16 is defined");
        Ok(g)
    };
    let hf_quant = dir.join("hf_quant_config.json");
    if let Ok(text) = fs::read(&hf_quant) {
        let v: serde_json::Value = serde_json::from_slice(&text)
            .map_err(|e| anyhow::anyhow!("{}: {e}", hf_quant.display()))?;
        let q = v.get("quantization").unwrap_or(&v);
        if algo_is_nvfp4(q) {
            return group_of(q).map(Some);
        }
    }
    let Some(q) = config
        .get("quantization_config")
        .or_else(|| config.get("text_config")?.get("quantization_config"))
    else {
        return Ok(None);
    };
    let ct_format = q.get("format").and_then(|v| v.as_str()).or_else(|| {
        q.get("config_groups")?
            .as_object()?
            .values()
            .find_map(|g| g.get("format")?.as_str())
    });
    anyhow::ensure!(
        ct_format != Some("nvfp4-pack-quantized"),
        "compressed-tensors nvfp4-pack-quantized checkpoints are not supported \
         (only NVIDIA ModelOpt NVFP4: weight / weight_scale / weight_scale_2)"
    );
    if algo_is_nvfp4(q) {
        return group_of(q).map(Some);
    }
    Ok(None)
}

/// Find a companion tensor by exact source name in any shard.
fn companion<'a>(files: &'a [SafeTensors], name: &str) -> Option<(&'a SafeTensors, &'a TensorMeta)> {
    files
        .iter()
        .find_map(|f| f.tensors.iter().find(|t| t.name == name).map(|t| (f, t)))
}

/// Decode one ModelOpt NVFP4 `.weight` (U8) with its `.weight_scale`
/// (F8_E4M3 per 16) and `.weight_scale_2` (F32 scalar) companions, folding an
/// optional AWQ `.pre_quant_scale` (`y = W·(x ⊙ s)`, so column i of W is
/// multiplied by `s[i]`). Returns `(shape, values)`.
fn nvfp4_decode(
    m: &TensorMeta,
    file: &SafeTensors,
    files: &[SafeTensors],
) -> anyhow::Result<(Vec<usize>, Vec<f32>)> {
    let name = &m.name;
    anyhow::ensure!(m.shape.len() == 2, "{name}: NVFP4 weight must be 2-D, got {:?}", m.shape);
    let (rows, cp) = (m.shape[0], m.shape[1]);
    let cols = cp * 2;
    let stem = name.strip_suffix(".weight").expect("suffix checked by caller");
    let (sf, sm) = companion(files, &format!("{stem}.weight_scale"))
        .ok_or_else(|| anyhow::anyhow!("{name}: NVFP4 weight without {stem}.weight_scale"))?;
    anyhow::ensure!(
        sm.dtype == "F8_E4M3" && sm.shape == vec![rows, cols / 16],
        "{name}: {stem}.weight_scale is {} {:?}, expected F8_E4M3 [{rows}, {}]",
        sm.dtype,
        sm.shape,
        cols / 16
    );
    let (gf, gm) = companion(files, &format!("{stem}.weight_scale_2"))
        .ok_or_else(|| anyhow::anyhow!("{name}: NVFP4 weight without {stem}.weight_scale_2"))?;
    let global = to_f32(&gm.dtype, gf.bytes(gm))?;
    anyhow::ensure!(
        global.len() == 1,
        "{name}: weight_scale_2 must be one value, got {}",
        global.len()
    );
    let mut vals = unpack_nvfp4(file.bytes(m), sf.bytes(sm), global[0], rows, cp)?;
    if let Some((pf, pm)) = companion(files, &format!("{stem}.pre_quant_scale")) {
        let pqs = to_f32(&pm.dtype, pf.bytes(pm))?;
        anyhow::ensure!(
            pqs.len() == cols,
            "{name}: pre_quant_scale has {} values for {cols} columns",
            pqs.len()
        );
        for row in vals.chunks_mut(cols) {
            for (w, s) in row.iter_mut().zip(&pqs) {
                *w *= s;
            }
        }
    }
    Ok((vec![rows, cols], vals))
}

/// The byte-identical MXFP4 path for a DeepSeek-V4/V4.1 source tensor: when
/// `m` is an `I8` FP4 weight `[rows, cols/2]` with an E8M0 `.scale`
/// companion `[rows, cols/32]` and at least one active output resolves this
/// tensor to `mxfp4` (the `fp4` profile's experts, or a `--tensor-quant`
/// override), returns the re-packed payload, its logical shape and, per
/// output, whether that output takes it. None = not such a tensor, or no
/// output wants it (the f32 decode then serves everyone as before).
#[allow(clippy::too_many_arguments)]
fn dsv4_raw_mxfp4(
    arch: &ModelArch,
    name: &str,
    m: &TensorMeta,
    file: &SafeTensors,
    files: &[SafeTensors],
    profiles: &[Quant],
    active: &[bool],
    no_defrag: bool,
) -> anyhow::Result<Option<(Vec<u8>, Vec<usize>, Vec<bool>)>> {
    if m.dtype != "I8" || m.shape.len() != 2 || name.ends_with(".scale") || !no_defrag {
        return Ok(None);
    }
    let (rows, cp) = (m.shape[0], m.shape[1]);
    let cols = cp * 2;
    if cols % 32 != 0 || force_f32(name) || force_f16(name) {
        return Ok(None);
    }
    let took: Vec<bool> = profiles
        .iter()
        .zip(active)
        .map(|(&q, &a)| a && effective_quant(arch, q, name) == Quant::Mxfp4)
        .collect();
    if !took.iter().any(|&t| t) {
        return Ok(None);
    }
    let scale_name = format!("{}.scale", m.name.trim_end_matches(".weight"));
    let Some((sf, sm)) = companion(files, &scale_name) else {
        return Ok(None);
    };
    anyhow::ensure!(
        matches!(sm.dtype.as_str(), "F8_E8M0" | "U8") && sm.shape == vec![rows, cols / 32],
        "{name}: FP4 source {} [{rows}, {cp}] with {scale_name} {} {:?} is not OCP MXFP4 \
         (expected E8M0 [{rows}, {}])",
        m.dtype,
        sm.dtype,
        sm.shape,
        cols / 32
    );
    let payload = cortiq_core::mxfp4::repack(file.bytes(m), sf.bytes(sm), rows, cols)
        .map_err(|e| anyhow::anyhow!("{name}: {e}"))?;
    Ok(Some((payload, vec![rows, cols], took)))
}

/// OCP FP8 E4M3 → f32. 1 sign, 4 exp (bias 7), 3 mantissa; no infinities,
/// and exp=15,mant=7 is the only NaN. Subnormals are m·2^-9.
#[inline]
pub(crate) fn fp8_e4m3_to_f32(b: u8) -> f32 {
    let sign = if b & 0x80 != 0 { -1.0f32 } else { 1.0 };
    let exp = ((b >> 3) & 0x0F) as i32;
    let man = (b & 0x07) as f32;
    // OCP E4M3FN reserves the all-ones exponent/mantissa encoding for NaN
    // (there are no infinities).  Treating 0x7f as a finite 480.0 silently
    // poisons every downstream quantization block.
    if exp == 15 && man == 7.0 {
        return f32::NAN;
    }
    let mag = if exp == 0 {
        man * (2f32).powi(-9)
    } else {
        (1.0 + man / 8.0) * (2f32).powi(exp - 7)
    };
    sign * mag
}

/// DeepSeek-V4 style FP8 weights: E4M3 values with one E8M0 scale per
/// `block`×`block` tile (`quantization_config.weight_block_size`). The
/// scale plane is `ceil(rows/block) × ceil(cols/block)`, row-major, and
/// the tiles at the edges are partial — the model's own layout, not a
/// padded one.
pub(crate) fn unpack_fp8_blocks(
    packed: &[u8],
    scales: &[u8],
    rows: usize,
    cols: usize,
    block: usize,
) -> anyhow::Result<Vec<f32>> {
    anyhow::ensure!(block > 0, "fp8 blocks: block size 0");
    let sr = rows.div_ceil(block);
    let sc = cols.div_ceil(block);
    anyhow::ensure!(
        packed.len() == rows * cols && scales.len() == sr * sc,
        "fp8 blocks: weight {} (want {}) / scales {} (want {}) for {rows}x{cols} block {block}",
        packed.len(),
        rows * cols,
        scales.len(),
        sr * sc
    );
    let mut out = vec![0.0f32; rows * cols];
    for r in 0..rows {
        let br = r / block;
        for c in 0..cols {
            let k = scales[br * sc + c / block];
            // E8M0: 2^(k−127); 255 is NaN per OCP — refuse loudly rather
            // than fold a NaN through the whole row.
            anyhow::ensure!(k != 255, "fp8 blocks: NaN scale at ({r},{c})");
            let v = fp8_e4m3_to_f32(packed[r * cols + c]);
            anyhow::ensure!(v.is_finite(), "fp8 blocks: NaN weight at ({r},{c})");
            out[r * cols + c] = v * (k as f32 - 127.0).exp2();
        }
    }
    Ok(out)
}

/// Fine-grained FP8 with one ordinary f32 inverse scale per source block.
/// V4.1's `weight_block_size` controls the block geometry; older checkpoints
/// use 128 while the V4.1 source uses 32.
fn unpack_fp8_scale_inv(
    packed: &[u8],
    scales: &[f32],
    rows: usize,
    cols: usize,
    block: usize,
) -> anyhow::Result<Vec<f32>> {
    anyhow::ensure!(block > 0, "fp8 scale_inv: block size 0");
    unpack_fp8_scale_inv_rows(
        packed,
        scales,
        rows,
        cols,
        block,
        rows.div_ceil(block),
        |r| r / block,
    )
}

/// [`unpack_fp8_scale_inv`] with an explicit weight-row → scale-row map, for
/// checkpoints whose scale grid is not one continuous tiling of the rows
/// (MiMo-V2's fused qkv restarts its 128-row blocks at every stored TP
/// chunk). `scale_rows` is the height of the scale plane; the column tiling
/// stays the ordinary `ceil(cols / block)`.
fn unpack_fp8_scale_inv_rows(
    packed: &[u8],
    scales: &[f32],
    rows: usize,
    cols: usize,
    block: usize,
    scale_rows: usize,
    scale_row: impl Fn(usize) -> usize + Sync,
) -> anyhow::Result<Vec<f32>> {
    anyhow::ensure!(block > 0, "fp8 scale_inv: block size 0");
    anyhow::ensure!(
        packed.len() == rows * cols,
        "fp8 scale_inv: weight size mismatch"
    );
    let (sr, sc) = (scale_rows, cols.div_ceil(block));
    anyhow::ensure!(
        scales.len() == sr * sc,
        "fp8 scale_inv: {} scales, expected {sr}x{sc}",
        scales.len()
    );
    // The map is checked once up front, so a bad map fails loudly instead
    // of panicking on an index inside a worker.
    if let Some(r) = (0..rows).find(|&r| scale_row(r) >= sr) {
        anyhow::bail!(
            "fp8 scale_inv: row {r} maps to scale row {} of {sr}",
            scale_row(r)
        );
    }
    let scale_row = &scale_row;
    let mut out = vec![0.0f32; rows * cols];
    let threads = encode_threads().min(rows.max(1)).max(1);
    let rows_per_thread = rows.div_ceil(threads);
    std::thread::scope(|scope| -> anyhow::Result<()> {
        let mut workers = Vec::with_capacity(threads);
        for (chunk_index, out_chunk) in out.chunks_mut(rows_per_thread * cols).enumerate() {
            let first_row = chunk_index * rows_per_thread;
            workers.push(scope.spawn(move || -> anyhow::Result<()> {
                for (local_row, out_row) in out_chunk.chunks_mut(cols).enumerate() {
                    let r = first_row + local_row;
                    let packed_row = &packed[r * cols..(r + 1) * cols];
                    let sr0 = scale_row(r) * sc;
                    let scale_plane_row = &scales[sr0..sr0 + sc];
                    for (c, (&byte, dst)) in packed_row.iter().zip(out_row).enumerate() {
                        let v = fp8_e4m3_to_f32(byte);
                        anyhow::ensure!(v.is_finite(), "fp8 scale_inv: NaN weight at ({r},{c})");
                        let scale = scale_plane_row[c / block];
                        anyhow::ensure!(
                            scale.is_finite(),
                            "fp8 scale_inv: non-finite scale at ({r},{c})"
                        );
                        *dst = v * scale;
                    }
                }
                Ok(())
            }));
        }
        for worker in workers {
            worker
                .join()
                .map_err(|_| anyhow::anyhow!("fp8 scale_inv worker panicked"))??;
        }
        Ok(())
    })?;
    Ok(out)
}

pub(crate) fn unpack_mlx(
    w_raw: &[u8],
    s_raw: &[u8],
    b_raw: Option<&[u8]>,
    out_dim: usize,
    in_dim: usize,
    bits: usize,
) -> anyhow::Result<Vec<f32>> {
    let mut out = vec![0f32; out_dim * in_dim];
    let num_groups = s_raw.len() / 2 / out_dim;
    let group_size = in_dim / num_groups;

    let w_u32: Vec<u32> = w_raw
        .chunks_exact(4)
        .map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]))
        .collect();
    let s_f16: Vec<u16> = s_raw
        .chunks_exact(2)
        .map(|b| u16::from_le_bytes([b[0], b[1]]))
        .collect();
    let b_f16: Option<Vec<u16>> = b_raw.map(|r| {
        r.chunks_exact(2)
            .map(|b| u16::from_le_bytes([b[0], b[1]]))
            .collect()
    });

    let vals_per_u32 = 32 / bits;
    let mask = (1 << bits) - 1;

    for row in 0..out_dim {
        for col in 0..in_dim {
            let group = col / group_size;
            let scale = f16_to_f32(s_f16[row * num_groups + group]);
            let bias = b_f16
                .as_ref()
                .map(|b| f16_to_f32(b[row * num_groups + group]))
                .unwrap_or(0.0);

            let u32_idx = (row * in_dim + col) / vals_per_u32;
            let shift = (col % vals_per_u32) * bits;
            let val = (w_u32[u32_idx] >> shift) & mask;

            // For 1-bit, MLX might map 0->-1 and 1->1, but wait!
            // In 2-bit, MLX maps 0,1,2,3 directly to value * scale + bias.
            // If the model is 1-bit, is the value 0 and 1, or is it sign bits?
            // Actually, bias handles the shift. If it's a 1-bit scale+bias model, `val * scale + bias` works.
            out[row * in_dim + col] = (val as f32) * scale + bias;
        }
    }
    Ok(out)
}

/// Stream the source MLX ternary planes into the production dtype16 Q2TP
/// affine profile.  This is intentionally not an experimental ternary dtype: dtype16
/// reserves rung 0 for an exact zero and has only 31 positive rungs, so row
/// parameters are refit against `q2tp_ladder` while the source 0/1/2 symbol
/// plane is copied unchanged.
fn encode_mlx_q2tp_affine(
    w_raw: &[u8],
    s_raw: &[u8],
    b_raw: &[u8],
    out_dim: usize,
    in_dim: usize,
) -> anyhow::Result<(Vec<u8>, f32)> {
    anyhow::ensure!(
        in_dim % 128 == 0,
        "Prism MLX input width {in_dim} is not g128"
    );
    anyhow::ensure!(
        w_raw.len() == out_dim * in_dim / 16 * 4,
        "Prism MLX packed weight length mismatch"
    );
    anyhow::ensure!(
        s_raw.len() == out_dim * (in_dim / 128) * 2,
        "Prism MLX scale length mismatch"
    );
    anyhow::ensure!(b_raw.len() == s_raw.len(), "Prism MLX bias length mismatch");
    let gpr = in_dim / GROUP_SIZE;
    let src_gpr = in_dim / 128;
    let stride = q4tp_code_stride(gpr);
    let mut chunks = vec![0u8; out_dim * gpr * Q2TP_CHUNK];
    let mut params = vec![0u8; out_dim * 4];
    let mut codes = vec![0u8; out_dim * stride];
    let bad_symbol = AtomicU8::new(0);
    let sum_sq = Mutex::new(0.0f64);
    let n_scales = AtomicUsize::new(0);
    encode_rows_parallel(
        out_dim,
        [gpr * Q2TP_CHUNK, 4, stride],
        [&mut chunks, &mut params, &mut codes],
        &|r, chunks_row, params_row, codes_row| {
            let mut source_scales = vec![F16_TINY; gpr];
            let mut lo = f32::INFINITY;
            let mut hi = f32::NEG_INFINITY;
            for g in 0..gpr {
                let sg = g / 4;
                let off = (r * src_gpr + sg) * 2;
                let s = f16_to_f32(u16::from_le_bytes([s_raw[off], s_raw[off + 1]]));
                let bias = f16_to_f32(u16::from_le_bytes([b_raw[off], b_raw[off + 1]]));
                if !s.is_finite() || s <= 0.0 {
                    bad_symbol.store(255, Ordering::Relaxed);
                    return;
                }
                if (bias + s).abs() > f32::max(2.0e-3, s.abs() * 2.0e-3) {
                    bad_symbol.store(254, Ordering::Relaxed);
                    return;
                }
                source_scales[g] = s;
                let lg = det_log2(s);
                lo = lo.min(lg);
                hi = hi.max(lg);
            }
            let lo_h = f32_to_f16(lo);
            let lo_r = f16_to_f32(lo_h);
            let mut st_h = f32_to_f16((hi - lo_r).max(0.0) / Q2TP_LMAX as f32);
            for _ in 0..64 {
                let st = f16_to_f32(st_h);
                if st > 0.0 && lo_r + Q2TP_LMAX as f32 * st >= hi {
                    break;
                }
                st_h += 1;
            }
            params_row[..2].copy_from_slice(&lo_h.to_le_bytes());
            params_row[2..].copy_from_slice(&st_h.to_le_bytes());
            let tab = det_q2tp_ladder(params_row, 0);
            let st = f16_to_f32(st_h);
            for g in 0..gpr {
                // Rung zero is reserved for an exact zero group in dtype16;
                // source ternary scales are positive, so use 1..=31.
                let c = if st > 0.0 {
                    1 + ((det_log2(source_scales[g]) - lo_r) / st)
                        .round_ties_even()
                        .clamp(0.0, Q2TP_LMAX as f32) as usize
                } else {
                    1
                };
                q4tp_put_code(codes_row, g, c);
                let fitted = tab[c];
                let d = fitted - source_scales[g];
                *sum_sq.lock().unwrap() += (d * d) as f64;
                n_scales.fetch_add(1, Ordering::Relaxed);
                let dst = &mut chunks_row[g * Q2TP_CHUNK..(g + 1) * Q2TP_CHUNK];
                for k in 0..Q2TP_CHUNK {
                    let col0 = g * GROUP_SIZE + k * 4;
                    let word0 = (r * (in_dim / 16) + col0 / 16) * 4;
                    let word = u32::from_le_bytes([
                        w_raw[word0],
                        w_raw[word0 + 1],
                        w_raw[word0 + 2],
                        w_raw[word0 + 3],
                    ]);
                    let mut b = 0u8;
                    for j in 0..4 {
                        let c = ((word >> (2 * ((col0 + j) % 16))) & 3) as u8;
                        if c == 3 {
                            bad_symbol.store(3, Ordering::Relaxed);
                        }
                        b |= c << (2 * j);
                    }
                    dst[k] = b;
                }
            }
        },
    );
    let bad = bad_symbol.load(Ordering::Relaxed);
    if bad != 0 {
        anyhow::bail!("Prism MLX source contains invalid scale or reserved code {bad}");
    }
    chunks.extend_from_slice(&params);
    chunks.extend_from_slice(&codes);
    let count = n_scales.load(Ordering::Relaxed);
    let rms = if count == 0 {
        0.0
    } else {
        (sum_sq.into_inner().unwrap() / count as f64).sqrt() as f32
    };
    Ok((chunks, rms))
}

/// Validate the source MLX packed plane before either Prism output profile is
/// written.  Code 3 is reserved by both CMF codecs; accepting it in the
/// ordinary q2tp candidate would make the two files disagree about the
/// source contract and would hide a corrupt checkpoint behind PTQ noise.
fn validate_mlx_ternary_codes(w_raw: &[u8], out_dim: usize, in_dim: usize) -> anyhow::Result<()> {
    anyhow::ensure!(
        in_dim % 16 == 0,
        "Prism MLX input width {in_dim} is not packed-g16"
    );
    let words_per_row = in_dim / 16;
    anyhow::ensure!(
        w_raw.len() == out_dim.saturating_mul(words_per_row).saturating_mul(4),
        "Prism MLX packed weight length mismatch"
    );
    for (word_index, raw) in w_raw.chunks_exact(4).enumerate() {
        let word = u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]);
        for j in 0..16 {
            if ((word >> (2 * j)) & 3) == 3 {
                let row = word_index / words_per_row;
                let col = (word_index % words_per_row) * 16 + j;
                anyhow::bail!(
                    "Prism MLX source contains reserved code 3 at row {row}, column {col}"
                );
            }
        }
    }
    Ok(())
}

/// Blob-layout sort key that puts tensors in decode-traversal order:
/// `(phase, layer, group, expert, projection)`. Phase orders embed → layers →
/// final-norm → lm_head → MTP → tail; within a layer, attention precedes the
/// FFN and MoE experts are grouped per expert (each expert's gate/up/down
/// contiguous). A stable name tiebreak keeps it deterministic. Layout only —
/// no effect on decoding (the directory is the offset authority).
pub(crate) fn exec_order_key(name: &str) -> (u32, u32, u32, u32, u32) {
    let num_after = |marker: &str| {
        name.split(marker)
            .nth(1)
            .and_then(|s| s.split('.').next())
            .and_then(|s| s.parse::<u32>().ok())
    };
    let expert = num_after(".experts.").unwrap_or(0);
    // Projection order within a block: q/gate, k/up, v/down, o, else.
    let proj = if name.contains("q_proj") || name.contains("gate_proj") {
        0
    } else if name.contains("k_proj") || name.contains("up_proj") {
        1
    } else if name.contains("v_proj") || name.contains("down_proj") {
        2
    } else if name.contains("o_proj") {
        3
    } else {
        4
    };
    if name.contains("embed_tokens") {
        (0, 0, 0, 0, 0)
    } else if let Some(l) = num_after(".layers.") {
        let group = if name.contains("input_layernorm") {
            0
        } else if name.contains("self_attn")
            || name.contains("linear_attn")
            || name.contains("short_conv")
        {
            1
        } else if name.contains("post_attention_layernorm") {
            2
        } else if name.ends_with("mlp.gate.weight")
            || name.contains("shared_expert")
            || name.contains("expert_bias")
        {
            3 // MoE router / shared expert (before the routed experts)
        } else if name.contains(".experts.") {
            4
        } else {
            5 // dense FFN (gate/up/down_proj) and anything else in the layer
        };
        (1, l, group, expert, proj)
    } else if name.contains("model.mtp") {
        (4, 0, 0, 0, 0)
    } else if name.contains("lm_head") {
        (3, 0, 0, 0, 0)
    } else if name.contains("model.norm") || name.ends_with("norm.weight") {
        (2, 0, 0, 0, 0)
    } else {
        (5, 0, 0, 0, 0)
    }
}

/// A tensor's metadata within a safetensors file (bytes are read lazily from mmap).
pub(crate) struct TensorMeta {
    pub(crate) name: String,
    pub(crate) dtype: String,
    pub(crate) shape: Vec<usize>,
    pub(crate) start: usize,
    pub(crate) end: usize,
}

/// A memory-mapped safetensors file — tensor bytes are borrowed from the mmap, so
/// the raw weights are never fully loaded into RAM (peak stays ~one tensor).
pub(crate) struct SafeTensors {
    mmap: Option<memmap2::Mmap>,
    data_start: usize,
    pub(crate) tensors: Vec<TensorMeta>,
    /// File name, so a resumed conversion can tell which of these are done.
    pub(crate) name: String,
}

impl SafeTensors {
    pub(crate) fn bytes(&self, m: &TensorMeta) -> &[u8] {
        let mmap = self
            .mmap
            .as_ref()
            .expect("safetensors mmap accessed after source shard release");
        &mmap[self.data_start + m.start..self.data_start + m.end]
    }

    /// Release the file mapping after this source shard's converted payloads
    /// and manifest checkpoint are durable. Metadata stays resident so the
    /// caller can retain the index for companion/name lookups.
    pub(crate) fn release(&mut self) {
        self.mmap.take();
    }
}

/// Release and unlink one local source shard after every active output has
/// checkpointed it. The index/config/tokenizer remain untouched; a missing
/// source file is reported as already consumed so a resumed output can finish
/// its cleanup pass without treating the absence as a conversion failure.
fn consume_source_shard(files: &mut [SafeTensors], index: usize, dir: &Path) -> anyhow::Result<()> {
    let source_name = files
        .get(index)
        .map(|f| f.name.clone())
        .ok_or_else(|| anyhow::anyhow!("source shard index {index} out of range"))?;
    files[index].release();
    let source_path = dir.join(&source_name);
    match fs::remove_file(&source_path) {
        Ok(()) => eprintln!("  consumed source shard {source_name}"),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
            eprintln!("  source shard {source_name} already absent")
        }
        Err(e) => {
            anyhow::bail!(
                "remove consumed source shard {}: {e}",
                source_path.display()
            )
        }
    }
    Ok(())
}

fn consume_local_ready_source(path: &Path) -> anyhow::Result<()> {
    fs::remove_file(path)
        .map_err(|e| anyhow::anyhow!("remove consumed local-ready source {}: {e}", path.display()))
}

pub(crate) fn open_safetensors(path: &Path) -> anyhow::Result<SafeTensors> {
    let file = fs::File::open(path).map_err(|e| anyhow::anyhow!("open {}: {e}", path.display()))?;
    let mmap = unsafe { memmap2::Mmap::map(&file)? };
    if mmap.len() < 8 {
        anyhow::bail!("{}: too small to be safetensors", path.display());
    }
    let hlen = u64::from_le_bytes(mmap[0..8].try_into().unwrap()) as usize;
    let header: serde_json::Value = serde_json::from_slice(&mmap[8..8 + hlen])?;
    let data_start = 8 + hlen;
    let obj = header
        .as_object()
        .ok_or_else(|| anyhow::anyhow!("bad safetensors header"))?;
    let mut tensors = Vec::new();
    for (name, v) in obj {
        if name == "__metadata__" {
            continue;
        }
        let dtype = v["dtype"].as_str().unwrap_or("").to_string();
        let shape: Vec<usize> = v["shape"]
            .as_array()
            .map(|a| a.iter().map(|x| x.as_u64().unwrap_or(0) as usize).collect())
            .unwrap_or_default();
        let offs = v["data_offsets"]
            .as_array()
            .ok_or_else(|| anyhow::anyhow!("tensor '{name}': no data_offsets"))?;
        let start = offs[0].as_u64().unwrap_or(0) as usize;
        let end = offs[1].as_u64().unwrap_or(0) as usize;
        tensors.push(TensorMeta {
            name: name.clone(),
            dtype,
            shape,
            start,
            end,
        });
    }
    Ok(SafeTensors {
        mmap: Some(mmap),
        data_start,
        tensors,
        name: path
            .file_name()
            .map(|n| n.to_string_lossy().into_owned())
            .unwrap_or_default(),
    })
}

/// Memory-map a model dir's weights (single file or sharded index).
pub(crate) fn open_model(dir: &Path) -> anyhow::Result<Vec<SafeTensors>> {
    open_model_filtered(dir, &std::collections::HashSet::new())
}

/// Return the source shard basename used in manifests and `SafeTensors.name`.
fn source_shard_name(path: &str) -> String {
    Path::new(path)
        .file_name()
        .map(|n| n.to_string_lossy().into_owned())
        .unwrap_or_else(|| path.to_string())
}

/// Move the explicitly requested source basenames to the front while keeping
/// every other validated shard in its original order.  The source list has
/// already come from the HF index (or the local source index) when this is
/// called, so an unknown name is a configuration error rather than a reason
/// to probe arbitrary files.  Empty, repeated, and unknown entries fail
/// closed before any streamed shard is opened.
fn apply_source_shard_priority(stream_shards: &mut Vec<String>) -> anyhow::Result<()> {
    let Some(raw) = std::env::var_os(SOURCE_SHARD_PRIORITY_ENV) else {
        return Ok(());
    };
    let raw = raw
        .to_str()
        .ok_or_else(|| anyhow::anyhow!("{SOURCE_SHARD_PRIORITY_ENV} must be valid UTF-8"))?;

    let known: HashSet<String> = stream_shards.iter().cloned().collect();
    anyhow::ensure!(
        known.len() == stream_shards.len(),
        "validated source shard list contains duplicate filenames"
    );
    let mut selected = HashSet::new();
    let mut priority = Vec::new();
    for name in raw.split(',') {
        anyhow::ensure!(
            !name.is_empty(),
            "{SOURCE_SHARD_PRIORITY_ENV} contains an empty entry"
        );
        anyhow::ensure!(
            known.contains(name),
            "{SOURCE_SHARD_PRIORITY_ENV} names unknown source shard {name:?}"
        );
        anyhow::ensure!(
            selected.insert(name.to_string()),
            "{SOURCE_SHARD_PRIORITY_ENV} repeats source shard {name:?}"
        );
        priority.push(name.to_string());
    }
    priority.extend(
        stream_shards
            .iter()
            .filter(|name| !selected.contains(name.as_str()))
            .cloned(),
    );
    *stream_shards = priority;
    Ok(())
}

/// Count tensors from the small safetensors headers/index without mapping the
/// payload. Resume uses this before opening source files so shards deleted by
/// a prior successful checkpoint can still contribute to progress and head
/// sizing.
fn source_shard_counts(dir: &Path) -> anyhow::Result<Vec<(String, usize)>> {
    let single = dir.join("model.safetensors");
    if single.exists() {
        return Ok(vec![(
            "model.safetensors".to_string(),
            safetensors_tensor_count(&single)?,
        )]);
    }
    let index = dir.join("model.safetensors.index.json");
    if index.exists() {
        let mut counts: HashMap<String, usize> = HashMap::new();
        let idx: serde_json::Value = serde_json::from_slice(&fs::read(&index)?)?;
        let map = idx["weight_map"]
            .as_object()
            .ok_or_else(|| anyhow::anyhow!("bad index json"))?;
        for file in map.values().filter_map(|v| v.as_str()) {
            *counts.entry(source_shard_name(file)).or_default() += 1;
        }
        let mut shards: Vec<(String, usize)> = counts.into_iter().collect();
        shards.sort_by(|a, b| a.0.cmp(&b.0));
        return Ok(shards);
    }
    anyhow::bail!(
        "no model.safetensors or model.safetensors.index.json in {}",
        dir.display()
    )
}

fn safetensors_tensor_count(path: &Path) -> anyhow::Result<usize> {
    let mut file =
        fs::File::open(path).map_err(|e| anyhow::anyhow!("open {}: {e}", path.display()))?;
    let mut len_bytes = [0u8; 8];
    file.read_exact(&mut len_bytes)?;
    let header_len = u64::from_le_bytes(len_bytes) as usize;
    let mut header_bytes = vec![0u8; header_len];
    file.read_exact(&mut header_bytes)?;
    let header: serde_json::Value = serde_json::from_slice(&header_bytes)
        .map_err(|e| anyhow::anyhow!("{}: bad safetensors header: {e}", path.display()))?;
    Ok(header
        .as_object()
        .map(|o| {
            o.keys()
                .filter(|name| name.as_str() != "__metadata__")
                .count()
        })
        .unwrap_or(0))
}

/// Open only source shards absent from the supplied completed-mark set. A
/// missing shard is therefore acceptable only when its basename is proven
/// complete by every output manifest; an unmarked missing shard remains a
/// hard error in `open_safetensors`.
pub(crate) fn open_model_filtered(
    dir: &Path,
    completed_shards: &std::collections::HashSet<String>,
) -> anyhow::Result<Vec<SafeTensors>> {
    let single = dir.join("model.safetensors");
    if single.exists() {
        if completed_shards.contains("model.safetensors") {
            return Ok(Vec::new());
        }
        return Ok(vec![open_safetensors(&single)?]);
    }
    let index = dir.join("model.safetensors.index.json");
    if index.exists() {
        let idx: serde_json::Value = serde_json::from_slice(&fs::read(&index)?)?;
        let map = idx["weight_map"]
            .as_object()
            .ok_or_else(|| anyhow::anyhow!("bad index json"))?;
        let mut files: Vec<String> = map
            .values()
            .filter_map(|v| v.as_str().map(String::from))
            .collect();
        files.sort();
        files.dedup();
        return files
            .iter()
            .filter(|f| !completed_shards.contains(&source_shard_name(f)))
            .map(|f| open_safetensors(&dir.join(f)))
            .collect();
    }
    anyhow::bail!(
        "no model.safetensors or model.safetensors.index.json in {}",
        dir.display()
    )
}

/// Read only completed source-shard marks from a manifest. This mirrors the
/// corruption boundary in `CmfStreamWriter::resume`: a malformed/truncated
/// line ends the durable prefix, so later marks are not trusted for filtering
/// source inputs.
fn resume_manifest_marks(path: &Path) -> std::collections::HashSet<String> {
    let Ok(text) = fs::read_to_string(path) else {
        return std::collections::HashSet::new();
    };
    let mut marks = std::collections::HashSet::new();
    for line in text.lines().skip(1) {
        if line.trim().is_empty() {
            continue;
        }
        let Ok(value) = serde_json::from_str::<serde_json::Value>(line) else {
            break;
        };
        if let Some(mark) = value["mark"].as_str() {
            marks.insert(mark.to_string());
        }
    }
    marks
}

fn cfg_usize(c: &serde_json::Value, key: &str) -> Option<usize> {
    c.get(key).and_then(|v| v.as_u64()).map(|x| x as usize)
}

/// Source FP8 tile width for DeepSeek-family checkpoints. V4.1 stores its
/// E4M3 weight and E8M0 scale plane over 32×32 tiles; older V4 files use the
/// 128×128 layout. Keep the source policy separate from the CMF quantizer,
/// since this value controls lossless decode before q4tp/q2tp encoding.
fn source_fp8_block(config: &serde_json::Value) -> usize {
    let model_type = config.get("model_type").and_then(|v| v.as_str());
    if model_type != Some("deepseek_v41") {
        return 128;
    }
    let block = config
        .get("quantization_config")
        .and_then(|q| q.get("weight_block_size"))
        .and_then(|v| v.as_array())
        .and_then(|a| {
            let row = a.first()?.as_u64()? as usize;
            let col = a.get(1)?.as_u64()? as usize;
            (row > 0 && row == col).then_some(row)
        });
    // The published V4.1 config carries [32, 32]. Keep the native default
    // when a compact test/config omits quantization_config entirely.
    block.unwrap_or(32)
}

// ── Xiaomi MiMo-V2 (`mimo_v2`) ───────────────────────────────────────────
//
// A hybrid MoE decoder: full-attention layers (config `num_key_value_heads`
// KV heads) interleaved with sliding-window layers (`swa_num_key_value_heads`
// KV heads, a window that includes the current token, and a learned per-head
// softmax sink). Q/K heads are `head_dim` wide with the first
// `head_dim·partial_rotary_factor` dims rotated; V heads are `v_head_dim`
// wide and scaled by `attention_value_scale`. The router is a sigmoid over
// the routed experts with a noaux selection bias; the first layer is dense.
//
// The release checkpoint stores q/k/v as ONE fused FP8 matrix per layer,
// pre-sharded for `num_key_value_heads` tensor-parallel ranks, with the FP8
// block scales tiled per shard — see `mimo_v2_split_qkv`. The routed experts
// are MXFP4 (U8 `.weight` + U8 E8M0 `.weight_scale`).

const MIMO_V2: &str = "mimo_v2";

/// Which towers of a MiMo-V2 checkpoint a conversion keeps
/// (`cortiq convert --mimo-towers`). The checkpoint ships a vision tower
/// (`visual.*`), an audio encoder (`audio_encoder.*`) and speech embeddings
/// (`speech_embeddings.*`) beside the text decoder, and the audio tokenizer
/// in its own `audio_tokenizer/` file. Names and codecs of the kept towers:
/// `cortiq_engine::mimo_mm` / `crate::mimo_towers`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MimoTowers {
    /// `text` (default): the text decoder alone; every tower is dropped.
    TextOnly,
    /// `mm-only`: the companion `<stem>.mm.cmf` — the towers and nothing
    /// of the text decoder (`mimo_towers::run_convert_mimo_mm`).
    MmOnly,
    /// `multimodal`: one file with the text decoder AND the towers under
    /// the companion's tensor names.
    Multimodal,
}

impl MimoTowers {
    pub fn label(self) -> &'static str {
        match self {
            Self::TextOnly => "text",
            Self::MmOnly => "mm-only",
            Self::Multimodal => "multimodal",
        }
    }
}

/// Parse `--mimo-towers text|mm-only|multimodal`.
pub fn parse_mimo_towers(s: &str) -> anyhow::Result<MimoTowers> {
    Ok(match s.trim().to_ascii_lowercase().as_str() {
        "text" | "text-only" => MimoTowers::TextOnly,
        "mm-only" | "mm_only" | "mm" => MimoTowers::MmOnly,
        "multimodal" | "full" => MimoTowers::Multimodal,
        other => anyhow::bail!("--mimo-towers '{other}': expected text, mm-only or multimodal"),
    })
}

/// MiMo-V2 tensor names → the CMF layout, or `None` to drop the tensor.
fn mimo_v2_canon(raw: &str, towers: MimoTowers) -> Option<String> {
    // The MTP head is three draft layers with their own fused qkv; the
    // runtime has no MiMo MTP (and the loader refuses a multi-layer head).
    // Plain greedy decoding is exact without it, so it is dropped in every
    // mode.
    if raw.starts_with("model.mtp.") {
        return None;
    }
    if ["visual.", "audio_encoder.", "speech_embeddings."]
        .iter()
        .any(|p| raw.starts_with(p))
    {
        return match towers {
            MimoTowers::TextOnly => None,
            MimoTowers::MmOnly | MimoTowers::Multimodal => Some(raw.to_string()),
        };
    }
    // The noaux selection bias lives under the router in the checkpoint;
    // the loader reads it as `mlp.expert_bias` (F32 via `force_f32`).
    // Without this rename the file converts and routes without the bias.
    if let Some(stem) = raw.strip_suffix(".mlp.gate.e_score_correction_bias") {
        return Some(format!("{stem}.mlp.expert_bias"));
    }
    // Per-head learned softmax sink of the sliding-window layers.
    if let Some(stem) = raw.strip_suffix(".self_attn.attention_sink_bias") {
        return Some(format!("{stem}.self_attn.sinks"));
    }
    canon_name(raw)
}

/// Header geometry of a `mimo_v2` config (validated).
#[derive(Clone, Debug, PartialEq)]
struct MimoV2Geometry {
    layer_types: Vec<LayerType>,
    kv_heads_per_layer: Vec<usize>,
    v_head_dim: usize,
    sliding_window: usize,
    swa_rope_theta: f64,
    rotary_dim: usize,
    rms_norm_eps: f64,
}

/// Read and validate the MiMo-V2 attention geometry. Every variant the
/// converter (or the runtime) does not implement is refused here, at the
/// config, rather than after a 300 GB download.
fn mimo_v2_geometry(
    tc: &serde_json::Value,
    n_layers: usize,
    n_heads: usize,
    head_dim: usize,
) -> anyhow::Result<MimoV2Geometry> {
    let flag = |k: &str| tc.get(k).and_then(|v| v.as_bool()).unwrap_or(false);
    let layout = tc
        .get("attention_projection_layout")
        .and_then(|v| v.as_str())
        .unwrap_or("<missing>");
    anyhow::ensure!(
        layout == "fused_qkv",
        "mimo_v2: attention_projection_layout '{layout}' is not supported (only fused_qkv)"
    );
    anyhow::ensure!(
        !flag("attention_bias"),
        "mimo_v2: attention_bias=true is not supported"
    );
    anyhow::ensure!(
        !flag("add_full_attention_sink_bias"),
        "mimo_v2: sinks on full-attention layers (add_full_attention_sink_bias) are not supported"
    );
    let same = |k: &str, want: usize| -> anyhow::Result<()> {
        if let Some(v) = tc.get(k).filter(|v| !v.is_null()) {
            let got = v
                .as_u64()
                .ok_or_else(|| anyhow::anyhow!("mimo_v2: {k} must be an integer"))?;
            anyhow::ensure!(
                got as usize == want,
                "mimo_v2: {k}={got} differs from the full-attention value {want}; \
                 per-layer head width/count is not supported"
            );
        }
        Ok(())
    };
    same("swa_head_dim", head_dim)?;
    same("swa_num_attention_heads", n_heads)?;
    let v_head_dim = cfg_usize(tc, "v_head_dim").unwrap_or(head_dim);
    same("swa_v_head_dim", v_head_dim)?;
    anyhow::ensure!(
        v_head_dim > 0 && v_head_dim <= head_dim,
        "mimo_v2: v_head_dim {v_head_dim} must be in 1..={head_dim}"
    );
    // The fused qkv is stored pre-sharded for `num_key_value_heads` ranks
    // (vLLM's ckpt_tp), so both KV head counts must split over it.
    let ckpt_tp = cfg_usize(tc, "num_key_value_heads")
        .ok_or_else(|| anyhow::anyhow!("mimo_v2: config missing num_key_value_heads"))?;
    let swa_kv = cfg_usize(tc, "swa_num_key_value_heads").unwrap_or(ckpt_tp);
    for (what, nkv) in [
        ("num_key_value_heads", ckpt_tp),
        ("swa_num_key_value_heads", swa_kv),
    ] {
        anyhow::ensure!(
            nkv > 0 && n_heads % nkv == 0 && nkv % ckpt_tp == 0 && n_heads % ckpt_tp == 0,
            "mimo_v2: {what}={nkv} must divide num_attention_heads={n_heads} and be a \
             multiple of the checkpoint's qkv shard count {ckpt_tp}"
        );
    }
    let pattern = tc
        .get("hybrid_layer_pattern")
        .and_then(|v| v.as_array())
        .ok_or_else(|| anyhow::anyhow!("mimo_v2: config missing hybrid_layer_pattern"))?;
    anyhow::ensure!(
        pattern.len() == n_layers,
        "mimo_v2: hybrid_layer_pattern has {} entries, expected {n_layers}",
        pattern.len()
    );
    let mut layer_types = Vec::with_capacity(n_layers);
    let mut kv_heads_per_layer = Vec::with_capacity(n_layers);
    for (li, v) in pattern.iter().enumerate() {
        match v.as_u64() {
            Some(0) => {
                layer_types.push(LayerType::FullAttention);
                kv_heads_per_layer.push(ckpt_tp);
            }
            Some(1) => {
                layer_types.push(LayerType::SlidingAttention);
                kv_heads_per_layer.push(swa_kv);
            }
            _ => anyhow::bail!("mimo_v2: hybrid_layer_pattern[{li}]={v} (expected 0 or 1)"),
        }
    }
    let window = cfg_usize(tc, "sliding_window")
        .or_else(|| cfg_usize(tc, "sliding_window_size"))
        .ok_or_else(|| anyhow::anyhow!("mimo_v2: config missing sliding_window"))?;
    if let (Some(a), Some(b)) = (
        cfg_usize(tc, "sliding_window"),
        cfg_usize(tc, "sliding_window_size"),
    ) {
        anyhow::ensure!(
            a == b,
            "mimo_v2: sliding_window {a} != sliding_window_size {b}"
        );
    }
    anyhow::ensure!(window > 0, "mimo_v2: sliding_window must be positive");
    let swa_rope_theta = tc
        .get("swa_rope_theta")
        .and_then(|v| v.as_f64())
        .ok_or_else(|| anyhow::anyhow!("mimo_v2: config missing swa_rope_theta"))?;
    // RoPE: default profile only, on the first int(head_dim·factor) dims.
    // The attention split reads the top-level factor and the rotary init
    // reads rope_parameters' — they must agree.
    let rope_params = tc.get("rope_parameters").filter(|v| !v.is_null());
    for r in [rope_params, tc.get("rope_scaling").filter(|v| !v.is_null())]
        .into_iter()
        .flatten()
    {
        let kind = r
            .get("rope_type")
            .or_else(|| r.get("type"))
            .and_then(|v| v.as_str())
            .unwrap_or("default");
        anyhow::ensure!(
            kind == "default",
            "mimo_v2: rope type '{kind}' is not supported (only default)"
        );
    }
    let top_prf = tc.get("partial_rotary_factor").and_then(|v| v.as_f64());
    let rope_prf = rope_params
        .and_then(|r| r.get("partial_rotary_factor"))
        .and_then(|v| v.as_f64());
    if let (Some(a), Some(b)) = (top_prf, rope_prf) {
        anyhow::ensure!(
            a == b,
            "mimo_v2: partial_rotary_factor {a} != rope_parameters.partial_rotary_factor {b}"
        );
    }
    let prf = top_prf.or(rope_prf).unwrap_or(1.0);
    // Python `int(head_dim * factor)`: 192·0.334 = 64.128 → 64.
    let rotary_dim = (head_dim as f64 * prf).floor() as usize;
    anyhow::ensure!(
        rotary_dim >= 2 && rotary_dim % 2 == 0 && rotary_dim <= head_dim,
        "mimo_v2: rotary dim {rotary_dim} (head_dim {head_dim} × {prf}) must be even and in 2..={head_dim}"
    );
    // Router: plain top-k over all experts (no group-limited routing).
    for k in ["n_group", "topk_group"] {
        if let Some(v) = tc.get(k).filter(|v| !v.is_null()) {
            anyhow::ensure!(
                v.as_u64() == Some(1),
                "mimo_v2: {k}={v} (group-limited routing) is not supported"
            );
        }
    }
    if let Some(m) = tc.get("topk_method").and_then(|v| v.as_str()) {
        anyhow::ensure!(
            m == "noaux_tc",
            "mimo_v2: topk_method '{m}' is not supported (only noaux_tc)"
        );
    }
    if let Some(s) = tc.get("scoring_func").and_then(|v| v.as_str()) {
        anyhow::ensure!(
            s == "sigmoid",
            "mimo_v2: scoring_func '{s}' is not supported (only sigmoid)"
        );
    }
    // Source quantization: the decoders below implement exactly FP8 E4M3
    // with 128×128 f32 inverse scales and MXFP4 experts in blocks of 32.
    let q = tc
        .get("quantization_config")
        .filter(|v| !v.is_null())
        .ok_or_else(|| {
            anyhow::anyhow!(
                "mimo_v2: no quantization_config — only the FP8/MXFP4 release checkpoint is supported"
            )
        })?;
    let qs = |k: &str| q.get(k).and_then(|v| v.as_str()).unwrap_or("<missing>");
    anyhow::ensure!(
        qs("quant_method") == "fp8" && qs("fmt") == "e4m3",
        "mimo_v2: source quantization {}/{} is not supported (only fp8/e4m3)",
        qs("quant_method"),
        qs("fmt")
    );
    let block: Vec<u64> = q
        .get("weight_block_size")
        .and_then(|v| v.as_array())
        .map(|a| a.iter().filter_map(|v| v.as_u64()).collect())
        .unwrap_or_default();
    anyhow::ensure!(
        block == [128, 128],
        "mimo_v2: FP8 weight_block_size {block:?} is not supported (only [128, 128])"
    );
    if let Some(store) = q.get("store_dtype").and_then(|v| v.as_str()) {
        anyhow::ensure!(
            store == "mxfp4",
            "mimo_v2: expert store_dtype '{store}' is not supported (only mxfp4)"
        );
        let mb = q
            .get("mxfp4_block_size")
            .and_then(|v| v.as_u64())
            .unwrap_or(32);
        anyhow::ensure!(
            mb == 32,
            "mimo_v2: mxfp4_block_size {mb} is not supported (only 32)"
        );
    }
    let rms_norm_eps = tc
        .get("layernorm_epsilon")
        .or_else(|| tc.get("rms_norm_eps"))
        .and_then(|v| v.as_f64())
        .unwrap_or(1e-6);
    Ok(MimoV2Geometry {
        layer_types,
        kv_heads_per_layer,
        v_head_dim,
        sliding_window: window,
        swa_rope_theta,
        rotary_dim,
        rms_norm_eps,
    })
}

/// `attention_value_scale`, folded into the V rows at conversion (the
/// attention output is linear in V, and HF scales V before the cache, so
/// the cached V matches the reference). Absent = 1.
fn mimo_v2_value_scale(tc: &serde_json::Value) -> anyhow::Result<f32> {
    match tc.get("attention_value_scale").filter(|v| !v.is_null()) {
        None => Ok(1.0),
        Some(v) => {
            let s = v.as_f64().ok_or_else(|| {
                anyhow::anyhow!("mimo_v2: attention_value_scale must be a number")
            })?;
            anyhow::ensure!(
                s.is_finite() && s > 0.0,
                "mimo_v2: attention_value_scale {s} must be finite and positive"
            );
            Ok(s as f32)
        }
    }
}

/// Row layout of one stored MiMo-V2 fused qkv: `ckpt_tp` chunks, each
/// `[Q_c | K_c | V_c]` holding that shard's `n_heads/ckpt_tp` query heads and
/// `n_kv_heads/ckpt_tp` key/value heads.
#[derive(Clone, Copy, Debug)]
struct MimoQkvLayout {
    n_heads: usize,
    n_kv_heads: usize,
    head_dim: usize,
    v_head_dim: usize,
    ckpt_tp: usize,
}

impl MimoQkvLayout {
    /// Q, K and V rows per stored chunk.
    fn per_chunk(&self) -> anyhow::Result<(usize, usize, usize)> {
        let t = self.ckpt_tp;
        anyhow::ensure!(
            t > 0 && self.n_heads % t == 0 && self.n_kv_heads % t == 0,
            "mimo_v2 qkv: {} q / {} kv heads do not split over {t} stored chunks",
            self.n_heads,
            self.n_kv_heads
        );
        Ok((
            self.n_heads / t * self.head_dim,
            self.n_kv_heads / t * self.head_dim,
            self.n_kv_heads / t * self.v_head_dim,
        ))
    }
}

/// Decode one stored MiMo-V2 fused qkv (FP8 E4M3 + f32 inverse block
/// scales) and split it into canonical `(q, k, v)` row-major matrices, with
/// `value_scale` folded into every V row.
///
/// Storage (vLLM `_shard_fp8_qkv_proj`, SGLang `_deinterleave_qkv_shards`):
/// `[Q_0|K_0|V_0 | Q_1|K_1|V_1 | …]`, one chunk per checkpoint TP rank, and
/// the 128-row scale blocks restart at every chunk: chunk `c` owns scale rows
/// `c·ceil(rpc/128) ..`, so `scale_row(r) = (r / rpc)·ceil(rpc/128) +
/// (r % rpc)/128`. On a full-attention layer a chunk is 3392 rows = 26.5
/// blocks, and the stored plane has 4·27 = 108 rows where one continuous
/// grid would have 106 — reading it continuously mis-scales most rows. The
/// continuous grid is accepted only when it is the same map (every chunk a
/// whole number of blocks). De-interleaving concatenates Q_c, K_c and V_c
/// over c, which keeps query head h on KV head h / (n_heads / n_kv_heads).
fn mimo_v2_split_qkv(
    packed: &[u8],
    scales: &[f32],
    scale_shape: &[usize],
    rows: usize,
    cols: usize,
    layout: MimoQkvLayout,
    block: usize,
    value_scale: f32,
) -> anyhow::Result<(Vec<f32>, Vec<f32>, Vec<f32>)> {
    let (q_pc, k_pc, v_pc) = layout.per_chunk()?;
    let t = layout.ckpt_tp;
    let rpc = q_pc + k_pc + v_pc;
    anyhow::ensure!(
        rows == t * rpc,
        "mimo_v2 qkv: {rows} rows, expected {t} chunks of {q_pc} q + {k_pc} k + {v_pc} v = {}",
        t * rpc
    );
    anyhow::ensure!(block > 0, "mimo_v2 qkv: block size 0");
    anyhow::ensure!(
        scale_shape.len() == 2 && scale_shape[1] == cols.div_ceil(block),
        "mimo_v2 qkv: scale plane {scale_shape:?} for {rows}x{cols} (block {block})"
    );
    let s_rows = scale_shape[0];
    let chunk_blocks = rpc.div_ceil(block);
    let chunked = s_rows == t * chunk_blocks;
    let continuous_same_map = s_rows == rows.div_ceil(block) && rpc % block == 0;
    anyhow::ensure!(
        chunked || continuous_same_map,
        "mimo_v2 qkv: scale plane has {s_rows} rows; expected {} ({t} chunks of {chunk_blocks} \
         blocks). A continuous {}-row grid over {rpc}-row chunks is a different layout and \
         would mis-scale the rows",
        t * chunk_blocks,
        rows.div_ceil(block)
    );
    let deq = unpack_fp8_scale_inv_rows(packed, scales, rows, cols, block, s_rows, |r| {
        (r / rpc) * chunk_blocks + (r % rpc) / block
    })?;
    let mut q = Vec::with_capacity(t * q_pc * cols);
    let mut k = Vec::with_capacity(t * k_pc * cols);
    let mut v = Vec::with_capacity(t * v_pc * cols);
    for c in 0..t {
        let base = c * rpc;
        q.extend_from_slice(&deq[base * cols..(base + q_pc) * cols]);
        k.extend_from_slice(&deq[(base + q_pc) * cols..(base + q_pc + k_pc) * cols]);
        v.extend(
            deq[(base + q_pc + k_pc) * cols..(base + rpc) * cols]
                .iter()
                .map(|&x| x * value_scale),
        );
    }
    Ok((q, k, v))
}

// ── MiMo-V2 MTP sidecar (`<stem>.mtp.cmf`) ───────────────────────────────
//
// The release ships `num_nextn_predict_layers` (3) draft layers in
// `model_mtp.safetensors`, 47 tensors, under `model.mtp.layers.N.*`. Each is
// one SWA transformer block with a dense MLP (vLLM `MiMoV2MTPLayer`, SGLang
// `MiMoV2ModelNextN`):
//     u = eh_proj·[enorm(embed(tok)); hnorm(hidden)]        embedding FIRST
//     u += o_proj(swa_attn(input_layernorm(u)))            sinks, window, narrow V
//     u += mlp(pre_mlp_layernorm(u))
//     logits = lm_head(final_layernorm(u))                 shared lm_head
// Their fused qkv uses the SWA geometry with the SAME TP-chunked layout and
// per-chunk FP8 scale grid as the backbone's sliding layers (14848 rows =
// 4 chunks × 3712, 116 = 4 × 29 scale rows), so `mimo_v2_split_qkv` decodes
// it unchanged. Embedding and lm_head are shared with the main model and are
// NOT written. The main text file never carries MTP (`mimo_v2_canon` drops
// `model.mtp.*`), so the draft layers live in a sidecar the loader finds by
// name (`cortiq_core::mtp_sidecar_path`) — a 164 GB main file never has to be
// rewritten to gain or drop them.

/// Canonical sidecar names: `pre_mlp_layernorm` becomes the loader's
/// `post_attention_layernorm`, the sink bias becomes `self_attn.sinks`;
/// everything else keeps its release name.
const MIMO_MTP_NORMS: [(&str, &str); 5] = [
    ("enorm", "enorm"),
    ("hnorm", "hnorm"),
    ("input_layernorm", "input_layernorm"),
    ("pre_mlp_layernorm", "post_attention_layernorm"),
    ("final_layernorm", "final_layernorm"),
];

/// Write the MiMo-V2 MTP sidecar for the checkpoint in `model_dir` (a local
/// release directory) to `output` (a main-file path is mapped to its
/// sidecar name). Attention, eh_proj and MLP matrices go to `quant` (the
/// skeleton policy: q8_2f); norms and sinks stay f32. Returns the path
/// written.
pub fn write_mimo_mtp_sidecar(
    model_dir: &Path,
    output: &Path,
    quant: &str,
) -> anyhow::Result<PathBuf> {
    let out = cortiq_core::mtp_sidecar_path(output);
    let config: serde_json::Value = serde_json::from_str(
        &fs::read_to_string(model_dir.join("config.json"))
            .map_err(|e| anyhow::anyhow!("{}: {e}", model_dir.join("config.json").display()))?,
    )?;
    let arch = build_arch(&config)?;
    anyhow::ensure!(
        arch.arch_name == MIMO_V2,
        "MTP sidecar: only mimo_v2 checkpoints are supported (got {})",
        arch.arch_name
    );
    let tc = config.get("text_config").unwrap_or(&config);
    let n_mtp = tc
        .get("num_nextn_predict_layers")
        .and_then(|v| v.as_u64())
        .unwrap_or(0) as usize;
    anyhow::ensure!(n_mtp > 0, "MTP sidecar: config has num_nextn_predict_layers = 0");
    let quant = parse_quant(quant)?;
    anyhow::ensure!(
        matches!(quant, Quant::Q8_2f | Quant::Q8Row | Quant::F16),
        "MTP sidecar: --quant must be q8_2f, q8 or f16 (the skeleton policy)"
    );
    // SWA geometry: the draft layers are sliding-window blocks.
    let li_swa = arch
        .layer_types
        .iter()
        .position(|t| matches!(t, LayerType::SlidingAttention))
        .ok_or_else(|| anyhow::anyhow!("MTP sidecar: the backbone has no sliding layer"))?;
    let kv = arch.kv_heads_per_layer.as_ref().ok_or_else(|| {
        anyhow::anyhow!("MTP sidecar: mimo_v2 arch without per-layer KV heads")
    })?;
    let (h, nh, hd) = (arch.hidden_size, arch.num_attention_heads, arch.head_dim);
    let vd = arch.v_head_dim.unwrap_or(hd);
    let layout = MimoQkvLayout {
        n_heads: nh,
        n_kv_heads: kv[li_swa],
        head_dim: hd,
        v_head_dim: vd,
        ckpt_tp: arch.num_kv_heads,
    };
    let inter = arch.intermediate_size;
    let value_scale = mimo_v2_value_scale(tc)?;
    let block = source_fp8_block(&config);

    // Source files: whatever the index maps model.mtp.* to (the release:
    // model_mtp.safetensors), else that file by name.
    let mut files: Vec<String> = Vec::new();
    let idx_path = model_dir.join("model.safetensors.index.json");
    if idx_path.exists() {
        let idx: serde_json::Value = serde_json::from_str(&fs::read_to_string(&idx_path)?)?;
        if let Some(map) = idx.get("weight_map").and_then(|m| m.as_object()) {
            for (name, file) in map {
                if let (true, Some(f)) = (name.starts_with("model.mtp."), file.as_str()) {
                    if !files.iter().any(|x| x == f) {
                        files.push(f.to_string());
                    }
                }
            }
        }
    }
    if files.is_empty() {
        files.push("model_mtp.safetensors".into());
    }
    let sources: Vec<SafeTensors> = files
        .iter()
        .map(|f| open_safetensors(&model_dir.join(f)))
        .collect::<anyhow::Result<_>>()?;
    let find = |name: &str| -> Option<(&SafeTensors, &TensorMeta)> {
        sources
            .iter()
            .find_map(|s| s.tensors.iter().find(|t| t.name == name).map(|t| (s, t)))
    };
    let need = |name: &str| {
        find(name).ok_or_else(|| anyhow::anyhow!("MTP sidecar: {name} not found in {files:?}"))
    };
    // A 2-D weight in any release storage: FP8 + scale_inv, or a float dtype.
    let dense = |name: &str, rows: usize, cols: usize| -> anyhow::Result<Vec<f32>> {
        let (src, m) = need(name)?;
        anyhow::ensure!(
            m.shape == [rows, cols],
            "MTP sidecar: {name} is {:?}, expected [{rows}, {cols}]",
            m.shape
        );
        if m.dtype == "F8_E4M3" {
            let sname = format!("{name}_scale_inv");
            let (ss, sm) = need(&sname)?;
            let scales = to_f32(&sm.dtype, ss.bytes(sm))?;
            unpack_fp8_scale_inv(src.bytes(m), &scales, rows, cols, block)
                .map_err(|e| anyhow::anyhow!("{name}: {e}"))
        } else {
            to_f32(&m.dtype, src.bytes(m))
        }
    };
    let vector = |name: &str, len: usize| -> anyhow::Result<Vec<f32>> {
        let (src, m) = need(name)?;
        anyhow::ensure!(
            m.shape.iter().product::<usize>() == len,
            "MTP sidecar: {name} is {:?}, expected {len} values",
            m.shape
        );
        to_f32(&m.dtype, src.bytes(m))
    };
    let f32_spec = |name: String, vals: &[f32]| TensorSpec {
        name,
        dtype: TensorDtype::F32,
        shape: vec![vals.len()],
        data: vals.iter().flat_map(|v| v.to_le_bytes()).collect(),
    };
    let mat_spec = |name: String, vals: &[f32], rows: usize, cols: usize| {
        let (dtype, data) = quantize_2d(quant, vals, rows, cols);
        TensorSpec {
            name,
            dtype,
            shape: vec![rows, cols],
            data,
        }
    };

    let mut tensors: Vec<TensorSpec> = Vec::new();
    for k in 0..n_mtp {
        let p = format!("model.mtp.layers.{k}.");
        for (src_n, dst_n) in MIMO_MTP_NORMS {
            let v = vector(&format!("{p}{src_n}.weight"), h)?;
            tensors.push(f32_spec(format!("{p}{dst_n}.weight"), &v));
        }
        let eh = dense(&format!("{p}eh_proj.weight"), h, 2 * h)?;
        tensors.push(mat_spec(format!("{p}eh_proj.weight"), &eh, h, 2 * h));

        let qkv = format!("{p}self_attn.qkv_proj.weight");
        let (src, m) = need(&qkv)?;
        anyhow::ensure!(
            m.dtype == "F8_E4M3" && m.shape.len() == 2 && m.shape[1] == h,
            "MTP sidecar: {qkv} must be a 2-D F8_E4M3 [rows, {h}], got {} {:?}",
            m.dtype,
            m.shape
        );
        let (ss, sm) = need(&format!("{qkv}_scale_inv"))?;
        let scales = to_f32(&sm.dtype, ss.bytes(sm))?;
        let (q, kk, v) = mimo_v2_split_qkv(
            src.bytes(m),
            &scales,
            &sm.shape,
            m.shape[0],
            h,
            layout,
            block,
            value_scale,
        )
        .map_err(|e| anyhow::anyhow!("{qkv}: {e}"))?;
        let nkv = layout.n_kv_heads;
        tensors.push(mat_spec(format!("{p}self_attn.q_proj.weight"), &q, nh * hd, h));
        tensors.push(mat_spec(format!("{p}self_attn.k_proj.weight"), &kk, nkv * hd, h));
        tensors.push(mat_spec(format!("{p}self_attn.v_proj.weight"), &v, nkv * vd, h));
        let o = dense(&format!("{p}self_attn.o_proj.weight"), h, nh * vd)?;
        tensors.push(mat_spec(format!("{p}self_attn.o_proj.weight"), &o, h, nh * vd));
        let sinks = vector(&format!("{p}self_attn.attention_sink_bias"), nh)?;
        anyhow::ensure!(
            sinks.iter().all(|s| s.is_finite()),
            "MTP sidecar: layer {k} has non-finite sinks"
        );
        tensors.push(f32_spec(format!("{p}self_attn.sinks"), &sinks));
        for (t, rows, cols) in [
            ("gate_proj", inter, h),
            ("up_proj", inter, h),
            ("down_proj", h, inter),
        ] {
            let w = dense(&format!("{p}mlp.{t}.weight"), rows, cols)?;
            tensors.push(mat_spec(format!("{p}mlp.{t}.weight"), &w, rows, cols));
        }
    }
    // Nothing in the MTP files may go unaccounted for: a tensor this writer
    // does not know is a layout it would silently drop.
    let known = |n: &str| {
        let Some(rest) = n.strip_prefix("model.mtp.layers.") else {
            return false;
        };
        let Some((li, tail)) = rest.split_once('.') else {
            return false;
        };
        li.parse::<usize>().is_ok_and(|li| li < n_mtp)
            && (MIMO_MTP_NORMS
                .iter()
                .any(|(s, _)| tail == format!("{s}.weight"))
                || [
                    "eh_proj.weight",
                    "self_attn.qkv_proj.weight",
                    "self_attn.qkv_proj.weight_scale_inv",
                    "self_attn.o_proj.weight",
                    "self_attn.o_proj.weight_scale_inv",
                    "self_attn.attention_sink_bias",
                    "eh_proj.weight_scale_inv",
                ]
                .contains(&tail)
                || ["gate_proj", "up_proj", "down_proj"].iter().any(|t| {
                    tail == format!("mlp.{t}.weight") || tail == format!("mlp.{t}.weight_scale_inv")
                }))
    };
    for s in &sources {
        for t in &s.tensors {
            if t.name.starts_with("model.mtp.") {
                anyhow::ensure!(
                    known(&t.name),
                    "MTP sidecar: unexpected tensor {} — refusing to drop it silently",
                    t.name
                );
            }
        }
    }

    let mut header_arch = arch.clone();
    header_arch.mtp = Some(cortiq_core::MtpConfig {
        num_layers: n_mtp,
        share_lm_head: true,
        share_embed: true,
    });
    let header = CmfHeader {
        format: "cmf".into(),
        version: CMF_VERSION,
        arch: header_arch,
        quant_type: match quant {
            Quant::Q8Row => QuantType::Q8Row,
            Quant::F16 => QuantType::F16,
            _ => QuantType::Q8_2f,
        },
        provenance: Some(serde_json::json!({
            "kind": "mimo_mtp_sidecar",
            "source": model_dir.display().to_string(),
            "source_files": files,
            "mtp_layers": n_mtp,
            "weight_quant": quant_name(quant),
            "attention_value_scale_folded": value_scale,
            "names": "model.mtp.layers.N.{enorm,hnorm,eh_proj,input_layernorm,self_attn.{q,k,v,o}_proj,self_attn.sinks,post_attention_layernorm (= pre_mlp_layernorm),mlp.{gate,up,down}_proj,final_layernorm}",
            "semantics": "u = eh_proj([enorm(embed(tok)); hnorm(hidden)]); SWA block (sinks, window, narrow V, swa theta); logits = lm_head(final_layernorm(u)); embed and lm_head shared with the main file",
            "converter": format!("cortiq {}", env!("CARGO_PKG_VERSION")),
        })),
        tokenizer_config: None,
        section_hashes: None,
        skills: Vec::new(),
        shard: None,
        calibration: None,
        routing: None,
        genome: None,
        lineage: Vec::new(),
        router: None,
        segments: Vec::new(),
    };
    if let Some(dir) = out.parent() {
        if !dir.as_os_str().is_empty() {
            fs::create_dir_all(dir)?;
        }
    }
    cortiq_core::CmfModel::write(&out, &header, &tensors, None, None)?;
    Ok(out)
}

// ── MiniCPM-V 4.7 DSpark sidecar (`<stem>.dspark.cmf`) ──────────────────
//
// DSpark is Qwen3 DFlash, not the MTP architecture carried by some CMF
// backbones.  In particular it has five dense Qwen3 blocks, receives a
// concatenation of five target hidden states, owns a separate KV cache, and
// applies a Markov correction over a 15-position masked draft block.  The
// generic `model.mtp.*` reader cannot safely interpret any of that.  P0
// therefore only packages the official checkpoint under an explicit
// `dspark.` namespace and binds it to one already-converted MiniCPM trunk.
// It deliberately does not activate speculative decoding.

const MINICPM_DSPARK_ARCH: &str = "minicpmv4_7_dspark_sidecar";
const MINICPM_DSPARK_SOURCE_ARCH: &str = "Qwen3DSparkModel";
const MINICPM_DSPARK_BASE_ARCH: &str = "minicpmv4_7";
const MINICPM_DSPARK_TARGET_IDS: [usize; 5] = [3, 11, 19, 27, 35];

/// Geometry of the one published MiniCPM-V 4.7 35B-A3B DSpark checkpoint.
/// Keeping this local to the converter is intentional: until a dedicated
/// executor exists, accepting a merely Qwen-shaped draft as this sidecar
/// would create a file that looks compatible but is not.
#[derive(Clone, Debug)]
struct DsparkLayout {
    hidden: usize,
    intermediate: usize,
    layers: usize,
    heads: usize,
    kv_heads: usize,
    head_dim: usize,
    vocab: usize,
    markov_rank: usize,
    block_size: usize,
    mask_token_id: usize,
    target_layers: [usize; 5],
    num_target_layers: usize,
    rope_theta: f64,
    rms_norm_eps: f64,
}

/// One tensor in the fixed DSpark payload schema.  `source` is deliberately
/// not a CMF backbone name: the writer prefixes every entry with `dspark.` so
/// it cannot accidentally be read as part of the target model.
#[derive(Clone, Debug)]
struct DsparkTensorSpec {
    source: String,
    shape: Vec<usize>,
    matrix: bool,
}

fn dspark_cfg_usize(config: &serde_json::Value, key: &str) -> anyhow::Result<usize> {
    config
        .get(key)
        .and_then(|v| v.as_u64())
        .map(|v| v as usize)
        .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: config has no integer {key}"))
}

fn dspark_cfg_bool(config: &serde_json::Value, key: &str) -> anyhow::Result<bool> {
    config
        .get(key)
        .and_then(|v| v.as_bool())
        .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: config has no boolean {key}"))
}

fn dspark_cfg_str<'a>(config: &'a serde_json::Value, key: &str) -> anyhow::Result<&'a str> {
    config
        .get(key)
        .and_then(|v| v.as_str())
        .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: config has no string {key}"))
}

impl DsparkLayout {
    /// Parse and fail closed on the public 35B-A3B DSpark contract.  The
    /// target-layer numbers stay in the source config's zero-based convention;
    /// a future runtime must make its capture convention explicit rather than
    /// silently treating the GGUF one-based presentation as equivalent.
    fn official(config: &serde_json::Value) -> anyhow::Result<Self> {
        let architectures = config
            .get("architectures")
            .and_then(|v| v.as_array())
            .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: config has no architectures array"))?;
        anyhow::ensure!(
            architectures.iter().any(|v| v.as_str() == Some(MINICPM_DSPARK_SOURCE_ARCH)),
            "DSpark sidecar: expected architectures to include {MINICPM_DSPARK_SOURCE_ARCH}"
        );
        anyhow::ensure!(
            dspark_cfg_str(config, "model_type")? == "qwen3",
            "DSpark sidecar: expected model_type qwen3"
        );
        let dflash = config
            .get("dflash_config")
            .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: config has no dflash_config"))?;
        anyhow::ensure!(
            dspark_cfg_str(dflash, "projector_type")? == "dspark",
            "DSpark sidecar: dflash_config.projector_type must be dspark"
        );
        anyhow::ensure!(
            dspark_cfg_str(dflash, "attention_mode")? == "gqa",
            "DSpark sidecar: dflash_config.attention_mode must be gqa"
        );
        anyhow::ensure!(
            dspark_cfg_str(dflash, "markov_head_type")? == "vanilla",
            "DSpark sidecar: dflash_config.markov_head_type must be vanilla"
        );
        anyhow::ensure!(
            dspark_cfg_bool(dflash, "enable_confidence_head")?
                && dspark_cfg_bool(dflash, "confidence_head_with_markov")?,
            "DSpark sidecar: the published confidence head contract is required"
        );
        let target_layers: Vec<usize> = dflash
            .get("target_layer_ids")
            .and_then(|v| v.as_array())
            .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: dflash_config has no target_layer_ids"))?
            .iter()
            .map(|v| {
                v.as_u64()
                    .map(|v| v as usize)
                    .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: target_layer_ids must be integers"))
            })
            .collect::<anyhow::Result<_>>()?;
        anyhow::ensure!(
            target_layers == MINICPM_DSPARK_TARGET_IDS,
            "DSpark sidecar: target_layer_ids {target_layers:?} are not the published MiniCPM-V 4.7 contract {:?}",
            MINICPM_DSPARK_TARGET_IDS
        );
        let target_layers: [usize; 5] = target_layers
            .try_into()
            .map_err(|_| anyhow::anyhow!("DSpark sidecar: expected exactly five target layers"))?;

        let layout = Self {
            hidden: dspark_cfg_usize(config, "hidden_size")?,
            intermediate: dspark_cfg_usize(config, "intermediate_size")?,
            layers: dspark_cfg_usize(config, "num_hidden_layers")?,
            heads: dspark_cfg_usize(config, "num_attention_heads")?,
            kv_heads: dspark_cfg_usize(config, "num_key_value_heads")?,
            head_dim: dspark_cfg_usize(config, "head_dim")?,
            vocab: dspark_cfg_usize(config, "vocab_size")?,
            markov_rank: dspark_cfg_usize(dflash, "markov_rank")?,
            block_size: dspark_cfg_usize(config, "block_size")?,
            mask_token_id: dspark_cfg_usize(dflash, "mask_token_id")?,
            target_layers,
            num_target_layers: dspark_cfg_usize(config, "num_target_layers")?,
            rope_theta: config
                .get("rope_parameters")
                .and_then(|v| v.get("rope_theta"))
                .and_then(|v| v.as_f64())
                .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: rope_parameters.rope_theta missing"))?,
            rms_norm_eps: config
                .get("rms_norm_eps")
                .and_then(|v| v.as_f64())
                .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: rms_norm_eps missing"))?,
        };
        anyhow::ensure!(
            layout.hidden == 2048
                && layout.intermediate == 6144
                && layout.layers == 5
                && layout.heads == 8
                && layout.kv_heads == 2
                && layout.head_dim == 256
                && layout.vocab == 248_144
                && layout.markov_rank == 256
                && layout.block_size == 15
                && layout.mask_token_id == 248_076
                && layout.num_target_layers == 40
                && (layout.rope_theta - 10_000_000.0).abs() < f64::EPSILON
                && (layout.rms_norm_eps - 1e-6).abs() < f64::EPSILON,
            "DSpark sidecar: unsupported MiniCPM DFlash geometry {layout:?}"
        );
        anyhow::ensure!(
            dspark_cfg_usize(config, "draft_vocab_size")? == layout.vocab,
            "DSpark sidecar: draft_vocab_size must equal vocab_size"
        );
        anyhow::ensure!(
            dspark_cfg_usize(config, "markov_rank")? == layout.markov_rank,
            "DSpark sidecar: root markov_rank disagrees with dflash_config"
        );
        anyhow::ensure!(
            dspark_cfg_bool(config, "enable_confidence_head")?
                && dspark_cfg_bool(config, "confidence_head_with_markov")?,
            "DSpark sidecar: root confidence-head flags disagree with dflash_config"
        );
        Ok(layout)
    }
}

fn dspark_tensor_specs(layout: &DsparkLayout) -> Vec<DsparkTensorSpec> {
    let mut specs = Vec::with_capacity(62);
    let mut add = |source: String, shape: Vec<usize>, matrix: bool| {
        specs.push(DsparkTensorSpec {
            source,
            shape,
            matrix,
        });
    };
    add(
        "fc.weight".into(),
        vec![layout.hidden, layout.hidden * layout.target_layers.len()],
        true,
    );
    add("hidden_norm.weight".into(), vec![layout.hidden], false);
    for li in 0..layout.layers {
        let p = format!("layers.{li}.");
        add(format!("{p}input_layernorm.weight"), vec![layout.hidden], false);
        add(
            format!("{p}self_attn.q_proj.weight"),
            vec![layout.heads * layout.head_dim, layout.hidden],
            true,
        );
        add(
            format!("{p}self_attn.k_proj.weight"),
            vec![layout.kv_heads * layout.head_dim, layout.hidden],
            true,
        );
        add(
            format!("{p}self_attn.v_proj.weight"),
            vec![layout.kv_heads * layout.head_dim, layout.hidden],
            true,
        );
        add(
            format!("{p}self_attn.o_proj.weight"),
            vec![layout.hidden, layout.heads * layout.head_dim],
            true,
        );
        add(format!("{p}self_attn.q_norm.weight"), vec![layout.head_dim], false);
        add(format!("{p}self_attn.k_norm.weight"), vec![layout.head_dim], false);
        add(
            format!("{p}post_attention_layernorm.weight"),
            vec![layout.hidden],
            false,
        );
        add(
            format!("{p}mlp.gate_proj.weight"),
            vec![layout.intermediate, layout.hidden],
            true,
        );
        add(
            format!("{p}mlp.up_proj.weight"),
            vec![layout.intermediate, layout.hidden],
            true,
        );
        add(
            format!("{p}mlp.down_proj.weight"),
            vec![layout.hidden, layout.intermediate],
            true,
        );
    }
    add("norm.weight".into(), vec![layout.hidden], false);
    add(
        "markov_head.markov_w1.weight".into(),
        vec![layout.vocab, layout.markov_rank],
        true,
    );
    add(
        "markov_head.markov_w2.weight".into(),
        vec![layout.vocab, layout.markov_rank],
        true,
    );
    add(
        "confidence_head.proj.weight".into(),
        vec![1, layout.hidden + layout.markov_rank],
        true,
    );
    add("confidence_head.proj.bias".into(), vec![1], false);
    debug_assert_eq!(specs.len(), 62);
    specs
}

fn dspark_canonical_name(source: &str) -> String {
    format!("dspark.{source}")
}

fn dspark_quant_type(quant: Quant) -> QuantType {
    match quant {
        Quant::F16 => QuantType::F16,
        Quant::Q8_2f => QuantType::Q8_2f,
        // QuantType predates q4tp; the directory carries the authoritative
        // Q4TiledP dtype and provenance names the exact codec.
        Quant::Q4TiledP => QuantType::Q4Block,
        _ => unreachable!("DSpark quant is validated before header construction"),
    }
}

fn dspark_matrix_dtype(quant: Quant) -> TensorDtype {
    match quant {
        Quant::F16 => TensorDtype::F16,
        Quant::Q8_2f => TensorDtype::Q8_2f,
        Quant::Q4TiledP => TensorDtype::Q4TiledP,
        _ => unreachable!("DSpark quant is validated before payload construction"),
    }
}

fn validate_dspark_source(
    source: &SafeTensors,
    specs: &[DsparkTensorSpec],
) -> anyhow::Result<()> {
    anyhow::ensure!(
        source.tensors.len() == specs.len(),
        "DSpark sidecar: source has {} tensors, expected {} — refusing to drop unknown tensors",
        source.tensors.len(),
        specs.len()
    );
    let known: HashSet<&str> = specs.iter().map(|s| s.source.as_str()).collect();
    for tensor in &source.tensors {
        anyhow::ensure!(
            known.contains(tensor.name.as_str()),
            "DSpark sidecar: unexpected source tensor {} — refusing to drop it",
            tensor.name
        );
    }
    for spec in specs {
        let tensor = source
            .tensors
            .iter()
            .find(|t| t.name == spec.source)
            .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: missing source tensor {}", spec.source))?;
        anyhow::ensure!(
            tensor.dtype == "BF16",
            "DSpark sidecar: {} is {}, expected BF16",
            spec.source,
            tensor.dtype
        );
        anyhow::ensure!(
            tensor.shape == spec.shape,
            "DSpark sidecar: {} has shape {:?}, expected {:?}",
            spec.source,
            tensor.shape,
            spec.shape
        );
        let elems = spec.shape.iter().product::<usize>();
        anyhow::ensure!(
            tensor.end >= tensor.start && tensor.end - tensor.start == elems * 2,
            "DSpark sidecar: {} has invalid BF16 byte span [{}, {}) for {:?}",
            spec.source,
            tensor.start,
            tensor.end,
            spec.shape
        );
    }
    Ok(())
}

#[derive(Clone, Debug)]
struct DsparkBaseBinding {
    filename: String,
    arch_name: String,
    trunk_hash: String,
    dir_hash: String,
}

fn validate_dspark_base(base_path: &Path, layout: &DsparkLayout) -> anyhow::Result<DsparkBaseBinding> {
    let filename = base_path
        .file_name()
        .and_then(|v| v.to_str())
        .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: --output must be a UTF-8 base .cmf filename"))?;
    anyhow::ensure!(
        filename.ends_with(".cmf")
            && !filename.ends_with(".mtp.cmf")
            && !filename.ends_with(".dspark.cmf"),
        "DSpark sidecar: --output must name the existing base .cmf, never an .mtp.cmf or .dspark.cmf"
    );
    let base = cortiq_core::CmfModel::open(base_path)
        .map_err(|e| anyhow::anyhow!("DSpark sidecar: open base {}: {e}", base_path.display()))?;
    let arch = base.arch();
    anyhow::ensure!(
        arch.arch_name == MINICPM_DSPARK_BASE_ARCH,
        "DSpark sidecar: base arch {} is not {}",
        arch.arch_name,
        MINICPM_DSPARK_BASE_ARCH
    );
    anyhow::ensure!(
        arch.num_layers == layout.num_target_layers
            && arch.hidden_size == layout.hidden
            && arch.vocab_size == layout.vocab
            && arch.head_dim == layout.head_dim,
        "DSpark sidecar: base geometry (layers {}, hidden {}, vocab {}, head {}) does not match draft target ({}, {}, {}, {})",
        arch.num_layers,
        arch.hidden_size,
        arch.vocab_size,
        arch.head_dim,
        layout.num_target_layers,
        layout.hidden,
        layout.vocab,
        layout.head_dim
    );
    for (name, shape) in [
        ("model.embed_tokens.weight", vec![layout.vocab, layout.hidden]),
        ("lm_head.weight", vec![layout.vocab, layout.hidden]),
    ] {
        let tensor = base.tensor(name).ok_or_else(|| {
            anyhow::anyhow!("DSpark sidecar: base is missing shared target tensor {name}")
        })?;
        anyhow::ensure!(
            tensor.shape == shape,
            "DSpark sidecar: base {name} shape {:?}, expected {:?}",
            tensor.shape,
            shape
        );
    }
    Ok(DsparkBaseBinding {
        filename: filename.to_string(),
        arch_name: arch.arch_name.clone(),
        trunk_hash: format!("{:016x}", base.trunk_hash()),
        dir_hash: format!("{:016x}", base.dir_hash()),
    })
}

fn dspark_sidecar_arch(layout: &DsparkLayout) -> anyhow::Result<ModelArch> {
    let config = serde_json::json!({
        "model_type": "qwen3",
        "hidden_size": layout.hidden,
        "intermediate_size": layout.intermediate,
        "num_hidden_layers": layout.layers,
        "num_attention_heads": layout.heads,
        "num_key_value_heads": layout.kv_heads,
        "head_dim": layout.head_dim,
        "vocab_size": layout.vocab,
        "max_position_embeddings": 262144,
        "rms_norm_eps": layout.rms_norm_eps,
        "rope_theta": layout.rope_theta,
        "hidden_act": "silu",
        "tie_word_embeddings": false,
    });
    let mut arch = build_arch(&config)?;
    arch.arch_name = MINICPM_DSPARK_ARCH.into();
    // The draft has no MTP, MoE, recurrent core, embedding, or lm_head of
    // its own.  Those last two are *bound* in provenance to the target file.
    arch.mtp = None;
    arch.moe = None;
    arch.linear_core = None;
    Ok(arch)
}

fn dspark_header(
    layout: &DsparkLayout,
    binding: &DsparkBaseBinding,
    model_dir: &Path,
    quant: Quant,
) -> anyhow::Result<CmfHeader> {
    Ok(CmfHeader {
        format: "cmf".into(),
        version: CMF_VERSION,
        arch: dspark_sidecar_arch(layout)?,
        quant_type: dspark_quant_type(quant),
        provenance: Some(serde_json::json!({
            "kind": "minicpm_v47_dspark_sidecar",
            "sidecar_format": "cmf-dspark-v1",
            "source": {
                "directory": model_dir.display().to_string(),
                "file": "model.safetensors",
                "architectures": [MINICPM_DSPARK_SOURCE_ARCH],
                "dtype": "BF16",
                "tensor_count": 62,
            },
            "base_binding": {
                "filename": binding.filename,
                "arch_name": binding.arch_name,
                "trunk_hash64": binding.trunk_hash,
                "dir_hash64": binding.dir_hash,
                "shared_target_tensors": {
                    "embedding": "model.embed_tokens.weight",
                    "lm_head": "lm_head.weight",
                },
            },
            "dflash": {
                "block_size": layout.block_size,
                "target_layer_ids_zero_based": layout.target_layers,
                "num_target_layers": layout.num_target_layers,
                "mask_token_id": layout.mask_token_id,
                "markov_rank": layout.markov_rank,
                "confidence_head": true,
                "confidence_head_with_markov": true,
                "capture_index_basis": "source config zero-based; executor must state its post-layer capture convention",
            },
            "tensor_namespace": "dspark.",
            "weight_quant": quant_name(quant),
            "matrix_policy": "every 2-D DSpark weight uses weight_quant; all norms and confidence bias are F32",
            "runtime": {
                "status": "conversion_only",
                "automatic_loading": false,
                "normal_target_behavior": "unchanged; a dedicated DFlash executor with exact target verification is required before use",
            },
            "converter": format!("cortiq {}", env!("CARGO_PKG_VERSION")),
        })),
        tokenizer_config: None,
        section_hashes: None,
        skills: Vec::new(),
        shard: None,
        calibration: None,
        routing: None,
        genome: None,
        lineage: Vec::new(),
        router: None,
        segments: Vec::new(),
    })
}

fn validate_written_dspark_sidecar(
    model: &cortiq_core::CmfModel,
    layout: &DsparkLayout,
    binding: &DsparkBaseBinding,
    quant: Quant,
) -> anyhow::Result<()> {
    anyhow::ensure!(
        model.arch().arch_name == MINICPM_DSPARK_ARCH,
        "DSpark sidecar: output arch identity was not preserved"
    );
    let provenance = model
        .header
        .provenance
        .as_ref()
        .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: output has no provenance"))?;
    anyhow::ensure!(
        provenance["kind"] == "minicpm_v47_dspark_sidecar"
            && provenance["sidecar_format"] == "cmf-dspark-v1"
            && provenance["tensor_namespace"] == "dspark.",
        "DSpark sidecar: output identity metadata is malformed"
    );
    anyhow::ensure!(
        provenance["base_binding"]["trunk_hash64"] == binding.trunk_hash
            && provenance["base_binding"]["dir_hash64"] == binding.dir_hash,
        "DSpark sidecar: output base binding changed while writing"
    );
    let specs = dspark_tensor_specs(layout);
    anyhow::ensure!(
        model.tensors.len() == specs.len(),
        "DSpark sidecar: output has {} tensors, expected {}",
        model.tensors.len(),
        specs.len()
    );
    for spec in specs {
        let name = dspark_canonical_name(&spec.source);
        let tensor = model
            .tensor(&name)
            .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: output is missing {name}"))?;
        anyhow::ensure!(
            tensor.shape == spec.shape,
            "DSpark sidecar: output {name} has shape {:?}, expected {:?}",
            tensor.shape,
            spec.shape
        );
        let expected_dtype = if spec.matrix {
            dspark_matrix_dtype(quant)
        } else {
            TensorDtype::F32
        };
        anyhow::ensure!(
            tensor.dtype == expected_dtype,
            "DSpark sidecar: output {name} is {:?}, expected {:?}",
            tensor.dtype,
            expected_dtype
        );
    }
    let problems = model.verify();
    anyhow::ensure!(
        problems.is_empty(),
        "DSpark sidecar: output hash verification failed: {}",
        problems.join("; ")
    );
    Ok(())
}

/// Encode a schema-validated DSpark source in one pass.  Kept separate from
/// publication so tests can exercise the exact tensor namespace/codec path on
/// a compact synthetic DFlash layout without relaxing the public checkpoint
/// validator.
fn write_dspark_payload(
    source: &SafeTensors,
    specs: &[DsparkTensorSpec],
    header: &CmfHeader,
    quant: Quant,
    output: &Path,
) -> anyhow::Result<()> {
    let mut writer = cortiq_core::format::CmfStreamWriter::new(
        output,
        cortiq_core::format::CmfStreamWriter::head_reserve_for(specs.len(), 64),
    )
    .map_err(|e| anyhow::anyhow!("DSpark sidecar: create {}: {e}", output.display()))?;
    for spec in specs {
        let tensor = source
            .tensors
            .iter()
            .find(|t| t.name == spec.source)
            .expect("source schema was validated");
        let vals = to_f32(&tensor.dtype, source.bytes(tensor))?;
        anyhow::ensure!(
            vals.len() == spec.shape.iter().product::<usize>() && vals.iter().all(|v| v.is_finite()),
            "DSpark sidecar: {} contains an invalid numeric payload",
            spec.source
        );
        let name = dspark_canonical_name(&spec.source);
        if spec.matrix {
            let (dtype, data) = quantize_2d(quant, &vals, spec.shape[0], spec.shape[1]);
            writer
                .push(&name, dtype, &spec.shape, &data)
                .map_err(|e| anyhow::anyhow!("DSpark sidecar: write {name}: {e}"))?;
        } else {
            let data: Vec<u8> = vals.iter().flat_map(|v| v.to_le_bytes()).collect();
            writer
                .push(&name, TensorDtype::F32, &spec.shape, &data)
                .map_err(|e| anyhow::anyhow!("DSpark sidecar: write {name}: {e}"))?;
        }
    }
    writer
        .finish(header, None, None)
        .map_err(|e| anyhow::anyhow!("DSpark sidecar: finalize {}: {e}", output.display()))
}

/// Convert the official MiniCPM-V-4.7-35B-A3B DSpark checkpoint to a
/// collision-free sidecar beside an existing CMF target.  This writes every
/// official draft tensor, validates the result, and binds it to the exact
/// base directory/trunk hashes.  It intentionally has no runtime effect.
pub fn write_minicpm_dspark_sidecar(
    model_dir: &Path,
    base_output: &Path,
    quant: &str,
) -> anyhow::Result<PathBuf> {
    let quant = parse_quant(quant)?;
    anyhow::ensure!(
        matches!(quant, Quant::Q8_2f | Quant::Q4TiledP | Quant::F16),
        "DSpark sidecar: --quant must be q8_2f, q4tp, or f16"
    );
    let config_path = model_dir.join("config.json");
    let config: serde_json::Value = serde_json::from_str(
        &fs::read_to_string(&config_path)
            .map_err(|e| anyhow::anyhow!("DSpark sidecar: {}: {e}", config_path.display()))?,
    )?;
    let layout = DsparkLayout::official(&config)?;
    let binding = validate_dspark_base(base_output, &layout)?;
    let out = cortiq_core::dspark_sidecar_path(base_output);
    anyhow::ensure!(
        out != base_output,
        "DSpark sidecar: refusing to reuse the base output path"
    );
    anyhow::ensure!(
        !out.exists(),
        "DSpark sidecar: {} already exists; refusing to overwrite it",
        out.display()
    );
    let source_path = model_dir.join("model.safetensors");
    let source = open_safetensors(&source_path)?;
    let specs = dspark_tensor_specs(&layout);
    validate_dspark_source(&source, &specs)?;
    let header = dspark_header(&layout, &binding, model_dir, quant)?;
    if let Some(parent) = out.parent().filter(|p| !p.as_os_str().is_empty()) {
        fs::create_dir_all(parent)?;
    }
    let filename = out
        .file_name()
        .and_then(|v| v.to_str())
        .ok_or_else(|| anyhow::anyhow!("DSpark sidecar: output filename must be UTF-8"))?;
    let partial = out.with_file_name(format!(".{filename}.part-{}", std::process::id()));
    let write_and_validate = (|| -> anyhow::Result<()> {
        write_dspark_payload(&source, &specs, &header, quant, &partial)?;
        let written = cortiq_core::CmfModel::open(&partial).map_err(|e| {
            anyhow::anyhow!("DSpark sidecar: reopen {} after write: {e}", partial.display())
        })?;
        validate_written_dspark_sidecar(&written, &layout, &binding, quant)
    })();
    if let Err(err) = write_and_validate {
        // This name belongs to this conversion only; cleaning it prevents a
        // failed 389M draft conversion from stranding a misleading payload or
        // consuming the disk budget for the next attempt.
        let _ = fs::remove_file(&partial);
        return Err(err);
    }
    fs::rename(&partial, &out).map_err(|e| {
        anyhow::anyhow!(
            "DSpark sidecar: publish {} -> {}: {e}",
            partial.display(),
            out.display()
        )
    })?;
    Ok(out)
}

/// Layer index of a `model.layers.N.*` name.
fn layer_index_of(name: &str) -> Option<usize> {
    name.strip_prefix("model.layers.")?
        .split('.')
        .next()?
        .parse()
        .ok()
}

/// The profile `cortiq convert` uses when `--quant` is not given (or is
/// `auto`): q8 for every family, except the large routed-expert MiMo-V2 and
/// Mellum profiles.  Those default to q4tp for the expert bank, while
/// `quant_for_tensor` keeps the always-active skeleton at q8_2f.
fn default_quant_for_arch(arch: &ModelArch) -> Quant {
    if arch.arch_name == MIMO_V2 || arch.arch_name.eq_ignore_ascii_case("mellum") {
        Quant::Q4TiledP
    } else {
        Quant::Q8Row
    }
}

/// `--quant` value that selects [`default_quant_for_arch`].
pub(crate) const AUTO_QUANT: &str = "auto";

/// Probe knob: `CMF_CONVERT_ONLY=glob[,glob…]` keeps only the source tensors
/// whose canonical name matches one of the `*`-globs (for a fused tensor, the
/// fused name). The result is a PARTIAL file — for checking a converter on a
/// few layers of a huge checkpoint, never for serving; its provenance says so.
const CONVERT_ONLY_ENV: &str = "CMF_CONVERT_ONLY";

fn parse_convert_only(spec: &str) -> anyhow::Result<Vec<String>> {
    let pats: Vec<String> = spec.split(',').map(|p| p.trim().to_string()).collect();
    anyhow::ensure!(
        !pats.is_empty() && pats.iter().all(|p| !p.is_empty()),
        "{CONVERT_ONLY_ENV}='{spec}': expected comma-separated non-empty globs"
    );
    Ok(pats)
}

/// Build ModelArch from a HF config.json (dense transformer families).
fn build_arch(config: &serde_json::Value) -> anyhow::Result<ModelArch> {
    // Whisper is encoder/decoder, so use its encoder dimensions for the
    // generic inspection fields and retain the full original config below.
    if config.get("model_type").and_then(|v| v.as_str()) == Some("whisper") {
        let get = |key: &str| {
            cfg_usize(config, key).ok_or_else(|| anyhow::anyhow!("whisper config: missing {key}"))
        };
        let hidden = get("d_model")?;
        let enc_heads = get("encoder_attention_heads")?;
        anyhow::ensure!(
            hidden > 0 && enc_heads > 0 && hidden % enc_heads == 0,
            "whisper config: d_model must be divisible by encoder_attention_heads"
        );
        anyhow::ensure!(
            get("num_mel_bins")? == 128,
            "whisper: only the 128-bin feature extractor is supported"
        );
        let synthetic = serde_json::json!({
            "model_type": "llama",
            "hidden_size": hidden,
            "intermediate_size": get("encoder_ffn_dim")?,
            "num_attention_heads": enc_heads,
            "num_key_value_heads": enc_heads,
            "num_hidden_layers": get("encoder_layers")?,
            "vocab_size": get("vocab_size")?,
            "max_position_embeddings": get("max_source_positions")?.checked_mul(2).unwrap_or(3000),
            "tie_word_embeddings": true,
        });
        let mut arch = build_arch(&synthetic)?;
        arch.arch_name = "whisper".into();
        arch.max_position_embeddings = get("max_source_positions")?;
        arch.hidden_act = "gelu".into();
        return Ok(arch);
    }
    // Vision/multimodal configs nest the text model under "text_config".
    let tc = config.get("text_config").unwrap_or(config);
    let model_type = config
        .get("model_type")
        .and_then(|v| v.as_str())
        .unwrap_or("unknown")
        .to_string();
    let is_prism = model_type == "prism_hadamard_qwen35";
    // Tencent HunYuan: the dense family (Hy-MT2-1.8B / 7B, `hunyuan_v1_dense`)
    // and the hy_v3 MoE (Hy-MT2-30B-A3B). Dense: Llama block + per-head qk
    // RMSNorm applied AFTER RoPE + NTK-alpha rope base. MoE: sigmoid router
    // with a selection bias, renormalized top-k × router_scaling_factor, one
    // shared expert, first layer dense.
    let is_hunyuan_dense = model_type == "hunyuan_v1_dense";
    if is_hunyuan_dense {
        anyhow::ensure!(
            !tc.get("use_cla").and_then(|v| v.as_bool()).unwrap_or(false),
            "hunyuan_v1_dense: cross-layer attention (use_cla) is not supported"
        );
        anyhow::ensure!(
            tc.get("norm_type").and_then(|v| v.as_str()).unwrap_or("rms") == "rms",
            "hunyuan_v1_dense: only RMS norms are supported"
        );
        anyhow::ensure!(
            !tc.get("add_classification_head")
                .and_then(|v| v.as_bool())
                .unwrap_or(false),
            "hunyuan_v1_dense: classification heads are not supported"
        );
        anyhow::ensure!(
            tc.get("use_qk_norm").and_then(|v| v.as_bool()).unwrap_or(true),
            "hunyuan_v1_dense: use_qk_norm=false is not supported"
        );
    }
    let is_dsv4 = model_type == "deepseek_v4";
    let is_dsv41 = model_type == "deepseek_v41";
    let tc_model_type = tc.get("model_type").and_then(|v| v.as_str()).unwrap_or("");
    // MiniCPM-V 4.7 wraps its Qwen3.5 text decoder in a vision model.  The
    // numerical contracts belong to the nested decoder, not its wrapper.
    let is_qwen35_text = model_type.to_ascii_lowercase().starts_with("qwen3_5")
        || tc_model_type.to_ascii_lowercase().starts_with("qwen3_5");
    // DeepSeek-V4: the name mapping and both source quantizations (FP8
    // E4M3 with 128x128 block scales, MXFP4 experts) are in place, but
    // five of its blocks have no runtime yet — and without them the file
    // would convert and then decode noise. Say exactly what is missing
    // here, at the config, rather than after a 167 GB download.
    if model_type == "deepseek_v4" {
        tracing::info!(
            "deepseek_v4: converting. The engine now carries all five of its \
             blocks — double-LoRA attention with a compressed KV, the per-layer \
             KV compressor, the sparse indexer, hyper-connections (Sinkhorn) and \
             hash routing — so this file is meant to DECODE, not merely to pack. \
             What has not been established is numerical parity against the \
             reference implementation: treat the first coherent generation as \
             the gate, not this message."
        );
    }
    let hidden = cfg_usize(tc, "hidden_size")
        .ok_or_else(|| anyhow::anyhow!("config: missing hidden_size"))?;
    let n_heads = cfg_usize(tc, "num_attention_heads")
        .ok_or_else(|| anyhow::anyhow!("config: missing num_attention_heads"))?;
    let n_layers = cfg_usize(tc, "num_hidden_layers")
        .ok_or_else(|| anyhow::anyhow!("config: missing num_hidden_layers"))?;
    // Linear-attention (GatedDeltaNet, Qwen3.5): the per-layer schedule comes
    // from config.layer_types; the vendor operator is carried 1:1 and we declare
    // the canonical core so the runtime dispatches it.
    let layer_types: Vec<LayerType> = match tc.get("layer_types").and_then(|v| v.as_array()) {
        Some(a) => a
            .iter()
            .map(|v| match v.as_str() {
                Some("linear_attention") => LayerType::LinearAttention,
                // LFM2 gated short convolution mixer.
                Some("conv") | Some("short_conv") => LayerType::ShortConv,
                Some("sliding_attention") => LayerType::SlidingAttention,
                _ => LayerType::FullAttention,
            })
            .collect(),
        None => vec![LayerType::FullAttention; n_layers],
    };
    // Kimi Linear / Kimi-K3: the per-layer schedule lives in
    // linear_attn_config.full_attn_layers (1-BASED layer numbers);
    // everything else is a KDA layer.
    let is_kimi =
        model_type == "kimi_linear" || model_type == "kimi_k3" || tc_model_type == "kimi_linear";
    let layer_types = if let Some(lac) = tc.get("linear_attn_config").filter(|_| is_kimi) {
        anyhow::ensure!(
            tc.get("attn_res_block_size")
                .map(|v| v.is_null())
                .unwrap_or(true),
            "kimi: attn_res_block_size residual streams (Kimi-K3) are not supported yet"
        );
        anyhow::ensure!(
            !tc.get("latent_moe_use_norm")
                .and_then(|v| v.as_bool())
                .unwrap_or(false),
            "kimi: latent MoE (routed_expert_up/down_proj, Kimi-K3) is not supported yet"
        );
        anyhow::ensure!(
            !tc.get("mla_use_output_gate")
                .and_then(|v| v.as_bool())
                .unwrap_or(false),
            "kimi: MLA output gate (Kimi-K3) is not supported yet"
        );
        let full: std::collections::HashSet<usize> = lac
            .get("full_attn_layers")
            .and_then(|v| v.as_array())
            .map(|a| {
                a.iter()
                    .filter_map(|v| v.as_u64())
                    .map(|v| v as usize)
                    .collect()
            })
            .unwrap_or_default();
        anyhow::ensure!(
            !full.is_empty(),
            "kimi: linear_attn_config.full_attn_layers is empty"
        );
        (1..=n_layers)
            .map(|i| {
                if full.contains(&i) {
                    LayerType::FullAttention
                } else {
                    LayerType::Kda
                }
            })
            .collect()
    } else {
        layer_types
    };
    let kimi_lac = tc.get("linear_attn_config").filter(|_| is_kimi);
    let has_linear = layer_types
        .iter()
        .any(|t| matches!(t, LayerType::LinearAttention));
    let lnv = cfg_usize(tc, "linear_num_value_heads");
    let lvd = cfg_usize(tc, "linear_value_head_dim");
    let linear_core = if has_linear {
        Some(LinearCoreConfig {
            kind: "gated_delta_net".into(),
            num_heads: lnv.unwrap_or(0),
            nphase: None,
            value_head_dim: lvd.unwrap_or(0),
            phase_delta_layers: None,
        })
    } else {
        None
    };
    // Qwen3.5 nests rope params under `rope_parameters`.  Laguna, Spark
    // and Mellum go one level deeper and carry independent full/sliding
    // profiles.  Mellum's full layers use YaRN while its sliding layers use
    // ordinary RoPE, so flattening this distinction would make a file load
    // but rotate one of the two layer classes with the wrong frequencies.
    let rope_root = tc.get("rope_parameters");
    // Spark-X2.5 nests its two profiles the same way.
    let is_laguna_config =
        model_type.eq_ignore_ascii_case("laguna") || model_type.eq_ignore_ascii_case("spark2_5");
    let is_mellum_config = model_type.eq_ignore_ascii_case("mellum");
    let is_dual_rope_config = is_laguna_config || is_mellum_config;
    let rope = if is_dual_rope_config {
        rope_root.and_then(|r| r.get("full_attention"))
    } else {
        rope_root
    };
    let local_rope = if is_dual_rope_config {
        rope_root.and_then(|r| r.get("sliding_attention"))
    } else {
        None
    };
    let rope_theta = tc
        .get("rope_theta")
        .and_then(|v| v.as_f64())
        .or_else(|| {
            rope.and_then(|r| r.get("rope_theta"))
                .and_then(|v| v.as_f64())
        })
        .unwrap_or(10_000.0);
    let prf = tc
        .get("partial_rotary_factor")
        .and_then(|v| v.as_f64())
        .or_else(|| {
            rope.and_then(|r| r.get("partial_rotary_factor"))
                .and_then(|v| v.as_f64())
        })
        .unwrap_or(1.0) as f32;
    let local_prf = local_rope
        .and_then(|r| r.get("partial_rotary_factor"))
        .and_then(|v| v.as_f64())
        .map(|v| v as f32);
    let attention_heads_per_layer = tc
        .get("num_attention_heads_per_layer")
        .and_then(|v| v.as_array())
        .map(|a| {
            a.iter()
                .map(|v| {
                    v.as_u64().map(|n| n as usize).ok_or_else(|| {
                        anyhow::anyhow!("num_attention_heads_per_layer must contain integers")
                    })
                })
                .collect::<anyhow::Result<Vec<_>>>()
        })
        .transpose()?;
    if let Some(heads) = &attention_heads_per_layer {
        anyhow::ensure!(
            heads.len() == n_layers,
            "num_attention_heads_per_layer has {} entries, expected {n_layers}",
            heads.len()
        );
        let nkv = cfg_usize(tc, "num_key_value_heads").unwrap_or(n_heads);
        anyhow::ensure!(
            heads.iter().all(|&nh| nh > 0 && nh % nkv == 0),
            "every per-layer attention head count must be positive and divisible by num_key_value_heads={nkv}"
        );
    }
    // Mixture-of-experts: the FFN becomes a router + per-expert matrices. Tensor
    // handling is unchanged (experts are ordinary 2-D matrices); we just declare
    // the MoE config so the runtime dispatches it. Router presence per layer
    // (in the directory) decides which layers are sparse.
    // DeepSeek-V2 MLA geometry (kv_lora_rank marks the family).
    let mla = cfg_usize(tc, "kv_lora_rank").map(|lora| cortiq_core::MlaConfig {
        kv_lora_rank: lora,
        qk_rope_head_dim: cfg_usize(tc, "qk_rope_head_dim").unwrap_or(64),
        qk_nope_head_dim: cfg_usize(tc, "qk_nope_head_dim").unwrap_or(128),
        v_head_dim: cfg_usize(tc, "v_head_dim").unwrap_or(128),
        q_lora_rank: cfg_usize(tc, "q_lora_rank"),
        // Kimi Linear: full-attention layers run NoPE (KDA carries
        // position) — the rotation is skipped, the layout is kept.
        nope: tc
            .get("mla_use_nope")
            .and_then(|v| v.as_bool())
            .unwrap_or(false),
    });
    let moe = tc
        .get("num_experts")
        .and_then(|v| v.as_u64())
        .or_else(|| tc.get("n_routed_experts").and_then(|v| v.as_u64()))
        .filter(|&n| n > 0)
        .map(|ne| {
            let mt = model_type.to_lowercase();
            let ntp_default = is_qwen35_text
                || is_prism
                || mt.contains("qwen3_next")
                || mt.contains("qwen4_exp")
                || mt.contains("gemma4");
            // LFM2-MoE routes with a sigmoid gate + selection bias (DeepSeek-V3
            // noaux_tc); Qwen keeps the softmax-over-all default.
            let is_lfm2 = mt.starts_with("lfm2");
            let is_laguna = mt == "laguna";
            MoeConfig {
                num_experts: ne as usize,
                top_k: cfg_usize(tc, "num_experts_per_tok")
                    .or_else(|| cfg_usize(tc, "top_k_experts"))
                    // Kimi spells it with the full word.
                    .or_else(|| cfg_usize(tc, "num_experts_per_token"))
                    .unwrap_or(2),
                moe_intermediate_size: cfg_usize(tc, "moe_intermediate_size").unwrap_or(0),
                norm_topk_prob: tc
                    .get("norm_topk_prob")
                    .and_then(|v| v.as_bool())
                    // Kimi: moe_renormalize is the same switch.
                    .or_else(|| tc.get("moe_renormalize").and_then(|v| v.as_bool()))
                    // HunYuan hy_v3: route_norm.
                    .or_else(|| tc.get("route_norm").and_then(|v| v.as_bool()))
                    .unwrap_or(ntp_default),
                shared_expert_intermediate_size: cfg_usize(tc, "shared_expert_intermediate_size")
                    .or_else(|| {
                        // DeepSeek fuses its n shared experts into one
                        // MLP of n·moe_intermediate_size (Kimi spells the
                        // count num_shared_experts).
                        Some(
                            cfg_usize(tc, "n_shared_experts")
                                .or_else(|| cfg_usize(tc, "num_shared_experts"))?
                                * cfg_usize(tc, "moe_intermediate_size")?,
                        )
                    }),
                router_sigmoid: is_lfm2
                    || is_laguna
                    || tc.get("scoring_func").and_then(|v| v.as_str()) == Some("sigmoid")
                    || tc
                        .get("moe_router_activation_func")
                        .and_then(|v| v.as_str())
                        == Some("sigmoid")
                    // HunYuan hy_v3: sigmoid scores, selection by score +
                    // expert_bias, weights from the unbiased scores.
                    || tc
                        .get("moe_router_use_sigmoid")
                        .and_then(|v| v.as_bool())
                        .unwrap_or(false),
                // A stored scale of 1.0 is the no-op default; only non-trivial
                // scales need to ride in the header.
                routed_scaling_factor: tc
                    .get("routed_scaling_factor")
                    .or_else(|| tc.get("moe_routed_scaling_factor"))
                    // HunYuan hy_v3 spells it router_scaling_factor.
                    .or_else(|| tc.get("router_scaling_factor"))
                    .and_then(|v| v.as_f64())
                    .map(|v| v as f32)
                    .filter(|&v| (v - 1.0).abs() > 1e-9),
                router_resonance: false,
            }
        });
    let head_dim = cfg_usize(tc, "head_dim").unwrap_or(hidden / n_heads.max(1));
    let mimo = if model_type == MIMO_V2 {
        Some(mimo_v2_geometry(tc, n_layers, n_heads, head_dim)?)
    } else {
        None
    };
    // Zero-centered RMSNorm x̂·(1+w): Gemma family and native HF
    // Qwen3.5 / Qwen3-Next checkpoints.  Prism/Bonsai is deliberately
    // excluded: its source is the already-sanitized MLX Qwen3.5 runtime
    // (plain nn.RMSNorm weights), so applying the Gemma +1 shift here would
    // add one a second time at every input/post/final/QK norm.
    let mt = model_type.to_lowercase();
    let is_laguna = mt == "laguna";
    let is_granite = mt == "granite";
    if is_laguna {
        anyhow::ensure!(
            !tc.get("swa_attention_sink_enabled")
                .and_then(|v| v.as_bool())
                .unwrap_or(false),
            "laguna: learned SWA attention sinks are not supported yet"
        );
        anyhow::ensure!(
            tc.get("moe_router_logit_softcapping")
                .and_then(|v| v.as_f64())
                .unwrap_or(0.0)
                == 0.0,
            "laguna: non-zero MoE router logit soft-capping is not supported"
        );
        anyhow::ensure!(
            !tc.get("moe_apply_router_weight_on_input")
                .and_then(|v| v.as_bool())
                .unwrap_or(false),
            "laguna: moe_apply_router_weight_on_input=true is not supported"
        );
    }
    if is_granite {
        let residual_multiplier = tc
            .get("residual_multiplier")
            .and_then(|v| v.as_f64())
            .unwrap_or(1.0);
        anyhow::ensure!(
            (residual_multiplier - 1.0).abs() <= f64::EPSILON,
            "granite: residual_multiplier={residual_multiplier} is not supported yet"
        );
    }
    let norm_style = if (mt.contains("gemma") && !mt.contains("gemma4") && !mt.contains("gemma3n"))
        || is_qwen35_text
        || mt.contains("qwen3_next")
        || mt.contains("qwen4_exp")
    {
        NormStyle::Gemma
    } else {
        // Gemma-4 went back to plain x̂·w (Gemma3nRMSNorm lineage).
        NormStyle::Qwen
    };
    // Gemma family: GeGLU FFN, √hidden embedding scale, an attention
    // scale of its own, and (Gemma-3) interleaved sliding-window layers
    // with a separate local RoPE base. Gemma-2's ATTENTION soft-capping
    // is not implemented — refuse it loudly rather than emit a wrong
    // file. (Gemma-4's FINAL-logit capping is supported.)
    let is_gemma = mt.contains("gemma");
    let is_gemma4 = mt.contains("gemma4");
    // Gemma-2: attention-logit soft-capping is a supported operator
    // (tanh(s/c)·c before the causal softmax); its every-other-layer
    // sliding schedule maps onto sliding_window_pattern = 2 (full
    // attention at odd indices), same machinery as gemma-3/Laguna.
    let is_gemma2 = mt.contains("gemma2");
    // Gemma-4 (text tower): plain x̂·w norms (unlike gemma-3), dual-geometry
    // attention (sliding GQA at head_dim + global MQA at global_head_dim
    // with proportional partial rotary), scale-less V-norm, per-layer
    // output scalars and final-logit capping. The dense 12B/31B variants
    // convert; the MoE / E-series machinery is refused honestly.
    if is_gemma4 {
        if cfg_usize(tc, "hidden_size_per_layer_input").unwrap_or(0) > 0 {
            anyhow::bail!(
                "{model_type}: gemma-4 E-series per-layer inputs are not supported yet — \
                 the dense 12B/31B variants convert natively"
            );
        }
        if cfg_usize(tc, "num_kv_shared_layers").unwrap_or(0) > 0 {
            anyhow::bail!("{model_type}: gemma-4 KV-shared layers are not supported yet");
        }
    }
    // Gemma-4 keys rope_parameters by layer type: the global layers'
    // theta is the model theta, the sliding layers' theta is the local
    // base (same split gemma-3 spells with flat keys).
    let (g4_rope_theta, g4_local_theta, g4_global_prf) = match rope {
        Some(r) if is_gemma4 => {
            let full = r.get("full_attention");
            let slide = r.get("sliding_attention");
            (
                full.and_then(|f| f.get("rope_theta"))
                    .and_then(|v| v.as_f64()),
                slide
                    .and_then(|f| f.get("rope_theta"))
                    .and_then(|v| v.as_f64()),
                full.and_then(|f| f.get("partial_rotary_factor"))
                    .and_then(|v| v.as_f64())
                    .map(|v| v as f32),
            )
        }
        _ => (None, None, None),
    };
    let rope_theta = g4_rope_theta.unwrap_or(rope_theta);
    // Both spellings are in the wild: newer configs say `rope_type`, older
    // ones (DeepSeek-V4 among them) say `type`. Reading only one of them
    // drops the profile silently and the model decodes with unscaled
    // frequencies — which looks like a bad conversion, not a missing field.
    let yarn = rope
        .filter(|r| {
            r.get("rope_type")
                .or_else(|| r.get("type"))
                .and_then(|v| v.as_str())
                == Some("yarn")
        })
        .map(|r| {
            Ok::<YarnConfig, anyhow::Error>(YarnConfig {
                factor: r
                    .get("factor")
                    .and_then(|v| v.as_f64())
                    .ok_or_else(|| anyhow::anyhow!("YaRN rope profile is missing factor"))?
                    as f32,
                original_max_position_embeddings: r
                    .get("original_max_position_embeddings")
                    .and_then(|v| v.as_u64())
                    .ok_or_else(|| {
                        anyhow::anyhow!(
                            "YaRN rope profile is missing original_max_position_embeddings"
                        )
                    })? as usize,
                beta_fast: r.get("beta_fast").and_then(|v| v.as_f64()).unwrap_or(32.0) as f32,
                beta_slow: r.get("beta_slow").and_then(|v| v.as_f64()).unwrap_or(1.0) as f32,
                mscale_all_dim: r
                    .get("mscale_all_dim")
                    .and_then(|v| v.as_f64())
                    .map(|v| v as f32),
                // Mellum gives the *full-attention* YaRN profile an explicit
                // 1.277258... cos/sin scale.  Preserve that source value;
                // the independent sliding profile is default RoPE and keeps
                // the pipeline's local scale at its 1.0 default.
                attention_factor: r
                    .get("attention_factor")
                    .and_then(|v| v.as_f64())
                    .unwrap_or_else(|| {
                        let factor = r.get("factor").and_then(|v| v.as_f64()).unwrap_or(1.0);
                        0.1 * factor.ln() + 1.0
                    }) as f32,
            })
        })
        .transpose()?;
    // Sliding pattern from the explicit layer-type list: full layers must
    // sit at every P-th position ((i+1) % P == 0), which is how the
    // runtime models the cadence.
    let g4_pattern: Option<usize> = if is_gemma4 {
        let fulls: Vec<usize> = tc
            .get("layer_types")
            .and_then(|v| v.as_array())
            .map(|a| {
                a.iter()
                    .enumerate()
                    .filter(|(_, v)| v.as_str() == Some("full_attention"))
                    .map(|(i, _)| i)
                    .collect()
            })
            .unwrap_or_default();
        let p = fulls.first().map(|f| f + 1).unwrap_or(0);
        if p == 0
            || fulls.iter().enumerate().any(|(k, &i)| i != p * (k + 1) - 1)
            || (n_layers / p) != fulls.len()
        {
            anyhow::bail!("{model_type}: irregular full/sliding layer schedule not supported");
        }
        Some(p)
    } else {
        None
    };
    let hidden_act = match tc
        .get("hidden_activation")
        .or_else(|| tc.get("hidden_act"))
        .and_then(|v| v.as_str())
        .unwrap_or("silu")
    {
        "gelu_pytorch_tanh" | "gelu_tanh" | "gelu_new" => "gelu_tanh".to_string(),
        // HF `gelu` is the exact erf form (Spark-X2.5).
        "gelu" => "gelu".to_string(),
        "silu" | "swish" => "silu".to_string(),
        other => anyhow::bail!("unsupported hidden_act '{other}'"),
    };
    let is_minicpm3 = model_type == "minicpm3";
    let embed_multiplier = if is_granite {
        tc.get("embedding_multiplier")
            .and_then(|v| v.as_f64())
            .unwrap_or(1.0) as f32
    } else if is_gemma {
        (hidden as f32).sqrt()
    } else if is_minicpm3 {
        // MiniCPM: embeddings enter the stack ×scale_emb.
        tc.get("scale_emb").and_then(|v| v.as_f64()).unwrap_or(1.0) as f32
    } else {
        1.0
    };
    // MiniCPM: logits = lm_head(h) · dim_model_base/hidden; the head is
    // tied to the embedding, so the divisor rides in the header.
    let logit_multiplier = if is_granite {
        let divisor = tc
            .get("logits_scaling")
            .and_then(|v| v.as_f64())
            .unwrap_or(1.0);
        anyhow::ensure!(
            divisor.is_finite() && divisor > 0.0,
            "granite: logits_scaling must be finite and positive"
        );
        let multiplier = (1.0 / divisor) as f32;
        ((multiplier - 1.0).abs() > f32::EPSILON).then_some(multiplier)
    } else if is_minicpm3 {
        cfg_usize(tc, "dim_model_base").map(|d| d as f32 / hidden as f32)
    } else {
        None
    };
    // Gemma-3n (E-series): AltUp/LAuReL/PLE/KV-sharing geometry rides
    // in the header; the runtime switches to the dedicated stack.
    let is_gemma3n = model_type.starts_with("gemma3n") || tc_model_type.starts_with("gemma3n");
    let g3n_cfg: Option<cortiq_core::G3nConfig> = if is_gemma3n {
        let need = |k: &str| {
            cfg_usize(tc, k).ok_or_else(|| anyhow::anyhow!("gemma3n: config missing {k}"))
        };
        Some(cortiq_core::G3nConfig {
            altup_num_inputs: need("altup_num_inputs")?,
            laurel_rank: need("laurel_rank")?,
            ple_dim: need("hidden_size_per_layer_input")?,
            ple_vocab: need("vocab_size_per_layer_input")?,
            num_kv_shared_layers: need("num_kv_shared_layers")?,
            activation_sparsity: tc
                .get("activation_sparsity_pattern")
                .and_then(|v| v.as_array())
                .map(|a| {
                    a.iter()
                        .filter_map(|v| v.as_f64())
                        .map(|v| v as f32)
                        .collect()
                })
                .unwrap_or_default(),
        })
    } else {
        None
    };
    let mut rope_freq_factors: Option<Vec<f64>> = None;
    // Phi-3 longrope: exact only within the ORIGINAL context — cap the
    // declared max honestly instead of serving stretched positions.
    let mut max_pos =
        cfg_usize(tc, "max_position_embeddings").unwrap_or(if is_kimi { 1_048_576 } else { 32768 });
    let mut ntk_theta: Option<f64> = None;
    if let Some(rs) = tc.get("rope_scaling").filter(|v| !v.is_null()) {
        let kind = rs
            .get("type")
            .or_else(|| rs.get("rope_type"))
            .and_then(|v| v.as_str());
        // HunYuan "dynamic" + alpha is NTK-alpha: ONE rescaled base,
        // base' = base · alpha^(d/(d−2)), fixed for every position — the
        // reference rotary module computes exactly this and never
        // rescales at inference. (Hy-MT2: 10000 · 1000^(128/126) =
        // 11 158 840, the value llama.cpp writes as rope.freq_base.)
        let ntk_alpha = rs
            .get("alpha")
            .and_then(|v| v.as_f64())
            .filter(|&a| a > 0.0 && (a - 1.0).abs() > 1e-12);
        match kind {
            Some("dynamic") if ntk_alpha.is_some() => {
                let alpha = ntk_alpha.unwrap();
                let d = head_dim as f64;
                let theta = rope_theta * alpha.powf(d / (d - 2.0));
                eprintln!(
                    "  rope: NTK-alpha {alpha} → base {theta:.1} (head_dim {head_dim}); \
                     the full {max_pos}-token window is native"
                );
                ntk_theta = Some(theta);
            }
            Some("longrope") | Some("su") | Some("yarn") | Some("linear") | Some("dynamic")
            | Some("mrope") => {
                let orig = cfg_usize(tc, "original_max_position_embeddings")
                    .or_else(|| {
                        rs.get("original_max_position_embeddings")
                            .and_then(|v| v.as_u64().map(|v| v as usize))
                    })
                    .unwrap_or(4096);
                eprintln!(
                    "  note: rope scaling '{:?}' — serving the exact {orig}-token native window",
                    kind.unwrap()
                );
                max_pos = orig;
                // MiniCPM3: the short factors are the TRAINED rope inside
                // the native window (1.06–16.9 per dim — not ≈1 like phi);
                // carry them so inv_freq divides per frequency at load.
                if is_minicpm3 {
                    if let Some(f) = rs.get("short_factor").and_then(|v| v.as_array()) {
                        let fac: Vec<f64> = f.iter().filter_map(|v| v.as_f64()).collect();
                        anyhow::ensure!(!fac.is_empty(), "minicpm3: empty longrope short_factor");
                        rope_freq_factors = Some(fac);
                    }
                }
            }
            Some(other) => anyhow::bail!("rope_scaling '{other}' not supported yet"),
            None => {}
        }
    }
    let mut arch = ModelArch {
        arch_name: model_type.clone(),
        hidden_size: hidden,
        intermediate_size: cfg_usize(tc, "intermediate_size")
            .or_else(|| {
                // Gemma-3n stores a per-layer LIST; E-models keep it
                // uniform — take the head and insist on uniformity.
                tc.get("intermediate_size")
                    .and_then(|v| v.as_array())
                    .and_then(|a| {
                        let first = a.first()?.as_u64()?;
                        a.iter()
                            .all(|v| v.as_u64() == Some(first))
                            .then_some(first as usize)
                    })
            })
            .or_else(|| cfg_usize(tc, "moe_intermediate_size"))
            .ok_or_else(|| anyhow::anyhow!("config: missing intermediate_size"))?,
        num_layers: n_layers,
        num_attention_heads: n_heads,
        num_kv_heads: cfg_usize(tc, "num_key_value_heads").unwrap_or(n_heads),
        head_dim,
        vocab_size: cfg_usize(tc, "vocab_size")
            .ok_or_else(|| anyhow::anyhow!("config: missing vocab_size"))?,
        layer_types,
        // LFM2 spells the RMSNorm epsilon `norm_eps`.
        rms_norm_eps: tc
            .get("rms_norm_eps")
            .or_else(|| tc.get("norm_eps"))
            .and_then(|v| v.as_f64())
            .unwrap_or(1e-6),
        norm_style,
        rope_theta: ntk_theta.unwrap_or(rope_theta),
        // Gemma ties embeddings by default and its configs omit the key.
        tie_word_embeddings: config
            .get("tie_word_embeddings")
            .and_then(|v| v.as_bool())
            // MiniCPM3 ships no lm_head tensor — the head is the embedding.
            .unwrap_or(is_gemma || is_minicpm3),
        // DeepSeek-V4 rotates only `qk_rope_head_dim` of each head's 512,
        // and its config states that directly rather than as a fraction.
        // Carrying it here is what lets a retuned checkpoint load without
        // the loader guessing.
        partial_rotary_factor: if is_dsv4 || is_dsv41 {
            match (cfg_usize(tc, "qk_rope_head_dim"), cfg_usize(tc, "head_dim")) {
                (Some(rd), Some(hd)) if hd > 0 => rd as f32 / hd as f32,
                _ => prf,
            }
        } else {
            prf
        },
        yarn,
        attention_heads_per_layer,
        // MTP head, when the checkpoint ships one. Qwen3.6 spells the count
        // `mtp_num_hidden_layers`; DeepSeek-lineage configs say
        // `num_nextn_predict_layers`. Absent → no speculative head, which is
        // the honest default for every model that has none.
        mtp: if mt.contains("qwen4_exp") {
            // qwen4_exp ships a second hyper-connected hybrid stack.  It is
            // not wire-compatible with the legacy DeepSeek/Qwen MTP block;
            // leave it out until that speculative-only head has an exact op.
            None
        } else if model_type == "minicpmv4_7" {
            // The public MiniCPM-V 4.7 checkpoints advertise one MTP layer
            // in config but ship no physical `mtp.*` tensors.  This converter
            // packages their language backbone only; carrying that orphaned
            // count into the CMF header would make the runtime advertise a
            // speculative head it cannot load.
            None
        } else {
            ["mtp_num_hidden_layers", "num_nextn_predict_layers"]
                .iter()
                .find_map(|k| cfg_usize(tc, k))
                .filter(|n| *n > 0)
                .map(|n| cortiq_core::MtpConfig {
                    num_layers: n,
                    share_lm_head: true,
                    share_embed: true,
                })
        },
        moe,
        qwen4_exp: (model_type == "qwen4_exp").then(|| Qwen4ExpConfig {
            hc_count: cfg_usize(tc, "hc_count").unwrap_or(4),
            hc_lowrank: cfg_usize(tc, "hc_lowrank").unwrap_or(320),
            indexer_n_heads: cfg_usize(tc, "indexer_n_heads").unwrap_or(4),
            indexer_kv_heads: cfg_usize(tc, "indexer_kv_heads").unwrap_or(1),
            indexer_head_dim: cfg_usize(tc, "indexer_head_dim").unwrap_or(128),
            indexer_budget: cfg_usize(tc, "indexer_budget").unwrap_or(2048),
            indexer_compress_ratio: cfg_usize(tc, "indexer_compress_ratio").unwrap_or(4),
            ple_layer_ids: tc
                .get("ple_layer_ids")
                .and_then(|v| v.as_array())
                .map(|a| {
                    a.iter()
                        .filter_map(|v| v.as_u64().map(|n| n as usize))
                        .collect()
                })
                .unwrap_or_else(|| vec![2]),
            ple_embed_dim: cfg_usize(tc, "ple_embed_dim").unwrap_or(hidden),
            ple_conv_kernel_size: cfg_usize(tc, "ple_conv_kernel_size").unwrap_or(4),
            ngram_size: cfg_usize(tc, "ngram_size").unwrap_or(3),
            heads_per_ngram: cfg_usize(tc, "heads_per_ngram").unwrap_or(8),
            ngram_vocab_size_base: cfg_usize(tc, "ngram_vocab_size_base").unwrap_or(20_000_000),
            make_ngram_vocab_size_divisible_by: cfg_usize(tc, "make_ngram_vocab_size_divisible_by")
                .unwrap_or(128),
            split_ngram_parts: cfg_usize(tc, "split_ngram_parts").unwrap_or(128),
            seed: cfg_usize(tc, "seed").unwrap_or(1234) as u64,
        }),
        // Keep the complete original HF object, including the text/vision
        // sub-configs and quantization policy. V4.1 runtime initialization
        // consumes this verbatim for multimodal and Engram geometry.
        deepseek_v41: is_dsv41.then(|| config.clone()),
        anchor_core: None,
        linear_core,
        head_clusters: None,
        max_position_embeddings: max_pos,
        // GDN spells it `linear_conv_kernel_dim`; LFM2 spells it
        // `conv_L_cache`; Kimi nests KDA geometry in linear_attn_config.
        linear_conv_kernel_dim: cfg_usize(tc, "linear_conv_kernel_dim")
            .or_else(|| cfg_usize(tc, "conv_L_cache"))
            .or_else(|| kimi_lac.and_then(|l| cfg_usize(l, "short_conv_kernel_size"))),
        linear_num_key_heads: cfg_usize(tc, "linear_num_key_heads")
            .or_else(|| kimi_lac.and_then(|l| cfg_usize(l, "num_heads"))),
        linear_num_value_heads: lnv,
        linear_key_head_dim: cfg_usize(tc, "linear_key_head_dim")
            .or_else(|| kimi_lac.and_then(|l| cfg_usize(l, "head_dim"))),
        linear_value_head_dim: lvd.or_else(|| kimi_lac.and_then(|l| cfg_usize(l, "head_dim"))),
        hidden_act,
        embed_multiplier,
        // Gemma-4 attends with scaling = 1.0 (q-norm carries the scale).
        query_pre_attn_scalar: tc
            .get("query_pre_attn_scalar")
            .and_then(|v| v.as_f64())
            // Granite specifies the DIRECT multiplier on QK^T.  CMF stores
            // the equivalent denominator s where the runtime uses 1/sqrt(s).
            .or_else(|| {
                tc.get("attention_multiplier")
                    .and_then(|v| v.as_f64())
                    .filter(|&v| v.is_finite() && v > 0.0)
                    .map(|v| 1.0 / (v * v))
            })
            // Gemma-4 and Gemma-3n attend with scale 1.0 (q-norm carries it).
            .or(if is_gemma4 || is_gemma3n {
                Some(1.0)
            } else {
                None
            }),
        sliding_window: cfg_usize(tc, "sliding_window").filter(|_| {
            is_laguna
                || mt == "spark2_5"
                || is_mellum_config
                || is_gemma2
                || is_gemma3n
                || tc.get("sliding_window_pattern").is_some()
                || g4_pattern.is_some()
        }),
        sliding_window_pattern: cfg_usize(tc, "sliding_window_pattern")
            .or(g4_pattern)
            .or(if is_gemma2 { Some(2) } else { None }),
        rope_local_base_freq: tc
            .get("rope_local_base_freq")
            .and_then(|v| v.as_f64())
            .or(g4_local_theta)
            .or_else(|| {
                local_rope
                    .and_then(|r| r.get("rope_theta"))
                    .and_then(|v| v.as_f64())
            }),
        local_partial_rotary_factor: local_prf,
        global_head_dim: cfg_usize(tc, "global_head_dim").filter(|_| is_gemma4),
        num_global_kv_heads: cfg_usize(tc, "num_global_key_value_heads").filter(|_| is_gemma4),
        global_partial_rotary_factor: g4_global_prf,
        final_logit_softcapping: tc.get("final_logit_softcapping").and_then(|v| v.as_f64()),
        attn_logit_softcapping: tc.get("attn_logit_softcapping").and_then(|v| v.as_f64()),
        mla,
        activation_situ_beta: tc.get("activation_situ_beta").and_then(|v| v.as_f64()),
        activation_situ_linear_beta: tc
            .get("activation_situ_linear_beta")
            .and_then(|v| v.as_f64()),
        attn_v_norm: is_gemma4,
        qk_norm_after_rope: is_hunyuan_dense,
        rope_freq_factors,
        logit_multiplier,
        g3n: g3n_cfg,
        // KDA (Kimi-K3): lower-bound decay-gate variant; absent = standard.
        kda_gate_lower_bound: tc
            .get("linear_attn_config")
            .and_then(|c| c.get("gate_lower_bound"))
            .and_then(|v| v.as_f64()),
        // Looped Transformer (Nanbeige 4.2): re-apply the layer stack num_loops times.
        num_loops: cfg_usize(tc, "num_loops").unwrap_or(1),
        // skip_loop_final_norm=false means loop_final_norm=true (apply norm after each loop).
        loop_final_norm: !tc
            .get("skip_loop_final_norm")
            .and_then(|v| v.as_bool())
            .unwrap_or(true),
        prism_hadamard: None,
        kv_heads_per_layer: None,
        v_head_dim: None,
    };
    if let Some(g) = mimo {
        // MiMo-V2: an explicit per-layer full/sliding schedule (the loader's
        // explicit_sliding path: window on the tagged layers, their own
        // local RoPE base), per-layer KV head counts, narrower V heads.
        arch.layer_types = g.layer_types;
        arch.kv_heads_per_layer = Some(g.kv_heads_per_layer);
        arch.v_head_dim = Some(g.v_head_dim);
        arch.sliding_window = Some(g.sliding_window);
        arch.sliding_window_pattern = None;
        arch.rope_local_base_freq = Some(g.swa_rope_theta);
        arch.local_partial_rotary_factor = None;
        // Store the rotary width as an exact ratio: the runtime computes
        // `(head_dim · factor) as usize`, and 64/192 lands on 64 by
        // construction instead of by the rounding of 192·0.334 = 64.128.
        let factor = g.rotary_dim as f32 / head_dim as f32;
        anyhow::ensure!(
            (head_dim as f32 * factor) as usize == g.rotary_dim,
            "mimo_v2: rotary factor {factor} does not reproduce {} dims",
            g.rotary_dim
        );
        arch.partial_rotary_factor = factor;
        arch.rms_norm_eps = g.rms_norm_eps;
        // The 3-layer MTP head is dropped at conversion (mimo_v2_canon).
        arch.mtp = None;
    }
    Ok(arch)
}

/// Load and validate the source-side Hadamard manifest for Prism/Bonsai.
/// Keeping it in the CMF header makes a converted file independently
/// executable; no `.signs` sidecar is consulted by the runtime.
fn load_prism_hadamard(
    dir: &Path,
    config: &serde_json::Value,
) -> anyhow::Result<Option<PrismHadamardConfig>> {
    if config.get("model_type").and_then(|v| v.as_str()) != Some("prism_hadamard_qwen35") {
        return Ok(None);
    }
    let rel = config
        .get("hadamard_config")
        .and_then(|v| v.as_str())
        .unwrap_or("hadamard.json");
    anyhow::ensure!(
        Path::new(rel).is_relative() && !rel.contains(".."),
        "invalid Prism hadamard_config path {rel:?}"
    );
    let raw = fs::read(dir.join(rel)).map_err(|e| {
        anyhow::anyhow!(
            "Prism checkpoint requires Hadamard manifest {}: {e}",
            dir.join(rel).display()
        )
    })?;
    let source: serde_json::Value = serde_json::from_slice(&raw)?;
    let widths: Vec<usize> = source
        .get("prism.hadamard.sign_widths")
        .and_then(|v| v.as_array())
        .ok_or_else(|| anyhow::anyhow!("Prism Hadamard manifest has no sign_widths"))?
        .iter()
        .map(|v| {
            v.as_u64()
                .map(|n| n as usize)
                .ok_or_else(|| anyhow::anyhow!("Prism Hadamard sign_widths must be integers"))
        })
        .collect::<anyhow::Result<_>>()?;
    let signs: Vec<f32> = source
        .get("prism.hadamard.sign_values")
        .and_then(|v| v.as_array())
        .ok_or_else(|| anyhow::anyhow!("Prism Hadamard manifest has no sign_values"))?
        .iter()
        .map(|v| {
            v.as_f64()
                .map(|n| n as f32)
                .ok_or_else(|| anyhow::anyhow!("Prism Hadamard sign_values must be numbers"))
        })
        .collect::<anyhow::Result<_>>()?;
    let cfg = PrismHadamardConfig {
        version: source
            .get("prism.hadamard.version")
            .and_then(|v| v.as_u64())
            .unwrap_or(0) as u32,
        block_size: source
            .get("prism.hadamard.block_size")
            .and_then(|v| v.as_u64())
            .unwrap_or(0) as usize,
        transform: source
            .get("prism.hadamard.transform")
            .and_then(|v| v.as_str())
            .unwrap_or_default()
            .to_string(),
        axis: source
            .get("prism.hadamard.axis")
            .and_then(|v| v.as_str())
            .unwrap_or_default()
            .to_string(),
        sign_mode: source
            .get("prism.hadamard.sign_mode")
            .and_then(|v| v.as_str())
            .unwrap_or_default()
            .to_string(),
        widths,
        signs,
        forward_weight_names: source
            .get("prism.hadamard.weight_names")
            .and_then(|v| v.as_array())
            .map(|a| {
                a.iter()
                    .filter_map(|v| v.as_str().map(String::from))
                    .collect()
            })
            .unwrap_or_default(),
        inverse_weight_names: source
            .get("prism.hadamard.inverse_weight_names")
            .and_then(|v| v.as_array())
            .map(|a| {
                a.iter()
                    .filter_map(|v| v.as_str().map(String::from))
                    .collect()
            })
            .unwrap_or_default(),
        gdn_v_grouped: source
            .get("prism.hadamard.gdn_v_grouped")
            .and_then(|v| v.as_bool())
            .unwrap_or(false),
        activation_f16: true,
        affine: None,
    };
    cfg.validate().map_err(|e| anyhow::anyhow!(e))?;
    Ok(Some(cfg))
}

/// Collect eos ids from generation_config.json / config.json (int or array).
fn eos_ids(gen_cfg: &serde_json::Value, config: &serde_json::Value) -> Vec<u32> {
    for v in [gen_cfg.get("eos_token_id"), config.get("eos_token_id")]
        .into_iter()
        .flatten()
    {
        if let Some(n) = v.as_u64() {
            return vec![n as u32];
        }
        if let Some(a) = v.as_array() {
            return a
                .iter()
                .filter_map(|x| x.as_u64().map(|n| n as u32))
                .collect();
        }
    }
    Vec::new()
}

/// `owner/name` HF repo id (not an existing local path).
pub(crate) fn looks_like_repo(s: &str) -> bool {
    let s = s.trim_matches('/');
    s.split('/').count() == 2 && !s.contains(char::is_whitespace) && !Path::new(s).exists()
}

/// A fresh ureq agent with the same timeouts the downloader uses.
fn hf_agent() -> ureq::Agent {
    ureq::AgentBuilder::new()
        .timeout_connect(Duration::from_secs(20))
        .timeout_read(Duration::from_secs(300))
        .build()
}

/// List a repo's files via the HF API (best-effort; empty on failure). Reused by
/// the GGUF importer to pick a `.gguf` from a repo.
pub(crate) fn hf_repo_files(repo: &str, token: Option<&str>) -> Vec<String> {
    repo_files(&hf_agent(), repo, token)
}

/// Download a single named file from an HF repo into the cache (parallel chunks
/// for large files); returns its local path. Used to fetch one `.gguf`.
pub(crate) fn hf_fetch_file(
    repo: &str,
    filename: &str,
    token: Option<&str>,
) -> anyhow::Result<std::path::PathBuf> {
    let dir = hf_cache_dir(repo)?;
    let dest = dir.join(filename.replace('/', "__"));
    let url = format!("https://huggingface.co/{repo}/resolve/main/{filename}");
    fetch(&hf_agent(), &url, &dest, token, true, hf_threads())?;
    Ok(dest)
}

/// Local cache dir for a downloaded HF repo (`~/.cache/cortiq/hf/owner--name`).
fn hf_cache_dir(repo: &str) -> anyhow::Result<std::path::PathBuf> {
    let base = std::env::var_os("HOME")
        .map(|h| std::path::PathBuf::from(h).join(".cache/cortiq/hf"))
        .unwrap_or_else(|| std::path::PathBuf::from(".cortiq-hf"));
    let dir = base.join(repo.replace('/', "--"));
    fs::create_dir_all(&dir)?;
    Ok(dir)
}

/// Parallel range chunk size (32 MiB), default connection count, and the
/// safety ceiling for user-selected parallelism.  The ceiling bounds worker
/// threads, simultaneous range buffers, and the corresponding HTTP sockets.
const HF_CHUNK: u64 = 32 * 1024 * 1024;
const HF_DEFAULT_THREADS: usize = 8;
const HF_MAX_THREADS: usize = 64;

fn parse_hf_threads(value: Option<&str>) -> usize {
    value
        .map(str::trim)
        .and_then(|v| v.parse::<usize>().ok())
        .filter(|&n| n >= 1)
        .unwrap_or(HF_DEFAULT_THREADS)
        .min(HF_MAX_THREADS)
}

fn hf_threads() -> usize {
    parse_hf_threads(std::env::var("CORTIQ_HF_THREADS").ok().as_deref())
}

/// Build disjoint half-open byte ranges for a remote file.  Keeping this as a
/// small pure helper makes the partitioning contract explicit and testable;
/// the downloader still uses one fixed 32 MiB range size, so each worker has a
/// bounded response buffer.
fn plan_hf_ranges(size: u64, chunk: u64) -> Vec<(u64, u64)> {
    if size == 0 || chunk == 0 {
        return Vec::new();
    }
    let mut ranges = Vec::new();
    let mut start = 0u64;
    while start < size {
        let end = start.saturating_add(chunk).min(size);
        ranges.push((start, end));
        start = end;
    }
    ranges
}

fn cached(dest: &Path) -> bool {
    dest.exists() && fs::metadata(dest).map(|m| m.len() > 0).unwrap_or(false)
}

fn auth(mut req: ureq::Request, token: Option<&str>) -> ureq::Request {
    req = req.set("User-Agent", "cortiq-convert");
    if let Some(t) = token {
        req = req.set("Authorization", &format!("Bearer {t}"));
    }
    req
}

/// Total size of a remote file via a `Range: bytes=0-0` probe (Content-Range),
/// or None if the server doesn't support/report ranges (→ single stream).
pub(crate) fn probe_size(agent: &ureq::Agent, url: &str, token: Option<&str>) -> Option<u64> {
    // A rate-limited probe would otherwise drop a 100 GB shard to the
    // single-stream path.
    let resp = with_retry(2, || {
        auth(agent.get(url).set("Range", "bytes=0-0"), token)
            .call()
            .map_err(|e| anyhow::anyhow!("probe {url}: {e}"))
    })
    .ok()?;
    resp.header("Content-Range")?
        .rsplit('/')
        .next()?
        .trim()
        .parse::<u64>()
        .ok()
}

fn get_range(
    agent: &ureq::Agent,
    url: &str,
    token: Option<&str>,
    start: u64,
    end: u64,
) -> anyhow::Result<Vec<u8>> {
    let resp = auth(
        agent
            .get(url)
            .set("Range", &format!("bytes={}-{}", start, end - 1)),
        token,
    )
    .call()
    .map_err(|e| anyhow::anyhow!("{e}"))?;
    let mut buf = Vec::with_capacity((end - start) as usize);
    resp.into_reader().read_to_end(&mut buf)?;
    Ok(buf)
}

fn write_at(path: &Path, offset: u64, data: &[u8]) -> std::io::Result<()> {
    use std::io::{Seek, SeekFrom, Write};
    let mut f = fs::OpenOptions::new().write(true).open(path)?;
    f.seek(SeekFrom::Start(offset))?;
    f.write_all(data)
}

/// `HF_TOKEN`, else `HUGGING_FACE_HUB_TOKEN` (the variables `hf` and
/// `huggingface_hub` read), when non-empty.
pub(crate) fn hf_token_from_env() -> Option<String> {
    ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]
        .iter()
        .find_map(|k| std::env::var(k).ok().filter(|t| !t.trim().is_empty()))
}

/// An HTTP answer the Hub sends when it is busy or rate limiting (429, 5xx):
/// worth waiting for, unlike a 404 or a refused token.
fn hub_busy(e: &anyhow::Error) -> bool {
    let s = format!("{e:#}");
    ["status code 429", "status code 500", "status code 502", "status code 503", "status code 504"]
        .iter()
        .any(|code| s.contains(code))
}

/// Retry `f` with exponential backoff — smooths over transient network errors.
/// A rate-limited or busy Hub (429/5xx) gets a much longer budget: pauses
/// from 10 s doubling to 2 min, for up to `CMF_HF_BUSY_WAIT_MIN` minutes
/// (default 30) — one 429 on the last shard must not end a streamed
/// conversion of hundreds of GB.
fn with_retry<T>(attempts: u32, mut f: impl FnMut() -> anyhow::Result<T>) -> anyhow::Result<T> {
    let mut delay = Duration::from_millis(400);
    let mut busy_delay = Duration::from_secs(10);
    let busy_budget = Duration::from_secs(
        60 * std::env::var("CMF_HF_BUSY_WAIT_MIN")
            .ok()
            .and_then(|v| v.parse::<u64>().ok())
            .unwrap_or(30),
    );
    let mut busy_waited = Duration::ZERO;
    let mut a = 0;
    loop {
        match f() {
            Ok(v) => return Ok(v),
            Err(e) if hub_busy(&e) && busy_waited + busy_delay <= busy_budget => {
                eprintln!(
                    "\n    hub busy ({e:#}); retrying in {} s",
                    busy_delay.as_secs()
                );
                std::thread::sleep(busy_delay);
                busy_waited += busy_delay;
                busy_delay = (busy_delay * 2).min(Duration::from_secs(120));
            }
            Err(e) => {
                a += 1;
                if a >= attempts {
                    return Err(e);
                }
                std::thread::sleep(delay);
                delay = (delay * 2).min(Duration::from_secs(8));
            }
        }
    }
}

/// Fetch one file into `dest` (cached). Large range-capable files are pulled in
/// parallel 32 MiB chunks over `threads` reused connections; otherwise a single
/// stream. Returns false on 404 when `required` is false.
pub(crate) fn fetch(
    agent: &ureq::Agent,
    url: &str,
    dest: &Path,
    token: Option<&str>,
    required: bool,
    threads: usize,
) -> anyhow::Result<bool> {
    // Keep the resource bound at the downloader boundary as well as in the
    // environment parser: all current callers use `hf_threads`, but this
    // prevents a future/internal caller from bypassing the 64-worker cap.
    let threads = threads.min(HF_MAX_THREADS);
    if cached(dest) {
        return Ok(true);
    }
    let tmp = dest.with_extension("part");
    let size = probe_size(agent, url, token);
    if let Some(sz) = size {
        if sz > HF_CHUNK && threads > 1 {
            {
                let f = fs::File::create(&tmp)?;
                f.set_len(sz)?;
            }
            let chunks = plan_hf_ranges(sz, HF_CHUNK);
            let total = chunks.len();
            let queue = Mutex::new(chunks);
            let err: Mutex<Option<String>> = Mutex::new(None);
            let done = std::sync::atomic::AtomicUsize::new(0);
            std::thread::scope(|scope| {
                // Do not create idle workers when a small file has only a
                // handful of ranges; this also keeps the resource bound
                // proportional to useful work.
                for _ in 0..threads.min(total) {
                    scope.spawn(|| {
                        loop {
                            if err.lock().unwrap().is_some() {
                                break;
                            }
                            let Some((start, end)) = queue.lock().unwrap().pop() else {
                                break;
                            };
                            // Each chunk retries on a transient failure before aborting.
                            let r = with_retry(4, || get_range(agent, url, token, start, end))
                                .and_then(|buf| write_at(&tmp, start, &buf).map_err(Into::into));
                            match r {
                                Ok(()) => {
                                    let d =
                                        done.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
                                    eprint!(
                                        "\r    downloading: {:>3}% ({d}/{total} chunks)",
                                        d * 100 / total
                                    );
                                }
                                Err(e) => {
                                    *err.lock().unwrap() = Some(e.to_string());
                                    break;
                                }
                            }
                        }
                    });
                }
            });
            eprintln!();
            if let Some(e) = err.into_inner().unwrap() {
                anyhow::bail!("download {url}: {e}");
            }
            fs::rename(&tmp, dest)?;
            return Ok(true);
        }
    }
    // Small file / no range support → single stream (with retry). Returns
    // Some(()) on success, None on an allowed 404 (optional file).
    let got = with_retry(4, || match auth(agent.get(url), token).call() {
        Ok(resp) => {
            let mut r = resp.into_reader();
            let mut f = fs::File::create(&tmp)?;
            std::io::copy(&mut r, &mut f)?;
            Ok(Some(()))
        }
        Err(ureq::Error::Status(404, _)) if !required => Ok(None),
        Err(e) => Err(anyhow::anyhow!("download {url}: {e}")),
    })?;
    match got {
        Some(()) => {
            fs::rename(&tmp, dest)?;
            Ok(true)
        }
        None => Ok(false),
    }
}

/// List a repo's file names via the HF API (best-effort; empty on any failure).
fn repo_files(agent: &ureq::Agent, repo: &str, token: Option<&str>) -> Vec<String> {
    let url = format!("https://huggingface.co/api/models/{repo}");
    match auth(agent.get(&url), token).call() {
        Ok(resp) => resp
            .into_json::<serde_json::Value>()
            .ok()
            .and_then(|j| {
                j["siblings"].as_array().map(|a| {
                    a.iter()
                        .filter_map(|s| s["rfilename"].as_str().map(String::from))
                        .collect()
                })
            })
            .unwrap_or_default(),
        Err(_) => Vec::new(),
    }
}

/// Fetch a HF repo's convertible files (config, tokenizer, weights) into the
/// cache, with parallel chunked downloads for the weight shards.
pub(crate) fn hf_download(repo: &str, token: Option<&str>) -> anyhow::Result<std::path::PathBuf> {
    Ok(hf_download_opts(repo, token, true)?.0)
}

/// `weights=false`: fetch only config/tokenizer/index and RETURN the
/// weight-shard names without downloading them — the streaming convert
/// pulls them one at a time (peak disk = one shard + the output).
pub(crate) fn hf_download_opts(
    repo: &str,
    token: Option<&str>,
    weights: bool,
) -> anyhow::Result<(std::path::PathBuf, Vec<String>)> {
    let dir = hf_cache_dir(repo)?;
    let base = format!("https://huggingface.co/{repo}/resolve/main");
    let threads = hf_threads();
    let agent = ureq::AgentBuilder::new()
        .timeout_connect(Duration::from_secs(20))
        .timeout_read(Duration::from_secs(300))
        .build();
    // config.json is mandatory for the safetensors path. If it is absent, give an
    // actionable message rather than a raw 404 — most often the repo is a GGUF-only
    // distribution (has `*.gguf`, no `config.json`), which needs a different tool.
    if !fetch(
        &agent,
        &format!("{base}/config.json"),
        &dir.join("config.json"),
        token,
        false,
        threads,
    )? {
        let files = repo_files(&agent, repo, token);
        let ggufs = files
            .iter()
            .filter(|f| f.to_lowercase().ends_with(".gguf"))
            .count();
        if ggufs > 0 {
            let src = repo
                .strip_suffix("-GGUF")
                .or_else(|| repo.strip_suffix("-gguf"))
                .filter(|s| !s.is_empty());
            anyhow::bail!(
                "'{repo}' is a GGUF repository ({ggufs} .gguf file(s), no config.json); \
                 `cortiq convert` needs a safetensors checkpoint. Either import a GGUF file \
                 directly with `cortiq import-gguf <file.gguf>` (dense llama/qwen2/qwen3, F32/F16/Q8_0), \
                 or convert the source safetensors repo instead{}.",
                match src {
                    Some(s) => format!(" — try `--model {s}`"),
                    None => String::new(),
                }
            );
        }
        anyhow::bail!("'{repo}': no config.json — not a Hugging Face safetensors checkpoint");
    }
    for (f, required) in [
        // Prism/Bonsai's signed FWHT descriptor is part of the checkpoint
        // identity.  It is optional for ordinary HF models and therefore
        // fetched best-effort here.
        ("hadamard.json", false),
        // NVIDIA ModelOpt checkpoints declare NVFP4 here rather than (or
        // as well as) in config.json's quantization_config.
        ("hf_quant_config.json", false),
        // Kimi ships tiktoken.model instead of tokenizer.json — either
        // satisfies the bundle (checked after conversion).
        ("tokenizer.json", false),
        ("tiktoken.model", false),
        ("tokenizer_config.json", false),
        ("generation_config.json", false),
        // Newer HF checkpoints (LFM2, Qwen3, …) ship the chat template as a
        // sidecar `chat_template.jinja` instead of embedding it in
        // tokenizer_config.json — without it `run` falls back to a generic
        // ChatML default that does not match the model's real format.
        ("chat_template.jinja", false),
    ] {
        fetch(
            &agent,
            &format!("{base}/{f}"),
            &dir.join(f),
            token,
            required,
            threads,
        )?;
    }
    let idx = dir.join("model.safetensors.index.json");
    if fetch(
        &agent,
        &format!("{base}/model.safetensors.index.json"),
        &idx,
        token,
        false,
        1,
    )? {
        let j: serde_json::Value = serde_json::from_slice(&fs::read(&idx)?)?;
        let map = j["weight_map"]
            .as_object()
            .ok_or_else(|| anyhow::anyhow!("bad safetensors index"))?;
        let mut shards: Vec<String> = map
            .values()
            .filter_map(|v| v.as_str().map(String::from))
            .collect();
        shards.sort();
        shards.dedup();
        if weights {
            for (i, s) in shards.iter().enumerate() {
                eprintln!(
                    "  shard {}/{} ({threads}× parallel): {s}",
                    i + 1,
                    shards.len()
                );
                fetch(
                    &agent,
                    &format!("{base}/{s}"),
                    &dir.join(s),
                    token,
                    true,
                    threads,
                )?;
            }
        }
        return Ok((dir, shards));
    } else if weights {
        eprintln!("  model.safetensors ({threads}× parallel)");
        fetch(
            &agent,
            &format!("{base}/model.safetensors"),
            &dir.join("model.safetensors"),
            token,
            true,
            threads,
        )?;
    }
    Ok((dir, vec!["model.safetensors".to_string()]))
}

/// Split a fused GDN projection (`in_proj_qkvz` or `in_proj_ba`) into the
/// canonical hub tensors. The fused weight is `[nk · group_width, hid]`; rows
/// are grouped by k-head. This mirrors transformers' `fix_query_key_value_ordering`
/// inverse — a pure row permutation, no value changes. Returns `(name, values,
/// out_rows)` for each produced tensor.
fn split_fused_gdn(
    name: &str,
    w: &[f32],
    hid: usize,
    nk: usize,
    dk: usize,
    nv: usize,
    dv: usize,
) -> anyhow::Result<Vec<(String, Vec<f32>, usize)>> {
    if nk == 0 || nv % nk != 0 {
        anyhow::bail!("fused GDN: bad head config nk={nk} nv={nv}");
    }
    let r = nv / nk;
    // Row `g·gw + gr` of the source (group g, within-group row gr).
    let row = |w: &[f32], gw: usize, g: usize, gr: usize| -> Vec<f32> {
        let base = (g * gw + gr) * hid;
        w[base..base + hid].to_vec()
    };

    if name.contains("in_proj_qkvz") {
        let gw = 2 * dk + 2 * r * dv;
        if w.len() != nk * gw * hid {
            anyhow::bail!(
                "fused GDN qkvz: {} values, expected {}",
                w.len(),
                nk * gw * hid
            );
        }
        // qkv = [q: nk·dk][k: nk·dk][v: nv·dv]
        let mut qkv = Vec::with_capacity((2 * nk * dk + nv * dv) * hid);
        for g in 0..nk {
            for rr in 0..dk {
                qkv.extend_from_slice(&row(w, gw, g, rr));
            }
        }
        for g in 0..nk {
            for rr in 0..dk {
                qkv.extend_from_slice(&row(w, gw, g, dk + rr));
            }
        }
        for g in 0..nk {
            for rr in 0..r * dv {
                qkv.extend_from_slice(&row(w, gw, g, 2 * dk + rr));
            }
        }
        // z = nv·dv
        let mut z = Vec::with_capacity(nv * dv * hid);
        for g in 0..nk {
            for rr in 0..r * dv {
                z.extend_from_slice(&row(w, gw, g, 2 * dk + r * dv + rr));
            }
        }
        let p = name.strip_suffix("in_proj_qkvz.weight").unwrap_or(name);
        Ok(vec![
            (format!("{p}in_proj_qkv.weight"), qkv, 2 * nk * dk + nv * dv),
            (format!("{p}in_proj_z.weight"), z, nv * dv),
        ])
    } else {
        // in_proj_ba: group width 2·r → b (first r per group), a (next r) → nv rows each.
        let gw = 2 * r;
        if w.len() != nk * gw * hid {
            anyhow::bail!(
                "fused GDN ba: {} values, expected {}",
                w.len(),
                nk * gw * hid
            );
        }
        let mut b = Vec::with_capacity(nv * hid);
        let mut a = Vec::with_capacity(nv * hid);
        for g in 0..nk {
            for rr in 0..r {
                b.extend_from_slice(&row(w, gw, g, rr));
            }
        }
        for g in 0..nk {
            for rr in 0..r {
                a.extend_from_slice(&row(w, gw, g, r + rr));
            }
        }
        let p = name.strip_suffix("in_proj_ba.weight").unwrap_or(name);
        Ok(vec![
            (format!("{p}in_proj_b.weight"), b, nv),
            (format!("{p}in_proj_a.weight"), a, nv),
        ])
    }
}

/// Convert a HF model (local directory or `owner/name` repo id) to a `.cmf`
/// file. `progress` receives fraction 0..1 (streamed as `@PROGRESS` markers).
// ───────────────────────── defrag (spec §11, Patent 2 claims 9/10) ─────────────────────────

enum FfnKind {
    Gate,
    Up,
    Down,
}

/// Match `model.layers.{li}.mlp.{gate|up|down}_proj.weight` → (layer, kind).
fn ffn_kind(name: &str) -> Option<(usize, FfnKind)> {
    let rest = name.strip_prefix("model.layers.")?;
    let dot = rest.find('.')?;
    let li: usize = rest[..dot].parse().ok()?;
    let kind = match &rest[dot + 1..] {
        "mlp.gate_proj.weight" => FfnKind::Gate,
        "mlp.up_proj.weight" => FfnKind::Up,
        "mlp.down_proj.weight" => FfnKind::Down,
        _ => return None,
    };
    Some((li, kind))
}

/// Drop dead neurons: gate/up keep ROWS (axis 0), down keeps COLUMNS
/// (axis 1). `keep` indexes the intermediate dim. Returns (reduced shape,
/// reduced f32 values).
fn slice_ffn(
    kind: &FfnKind,
    shape: &[usize],
    vals: &[f32],
    keep: &[bool],
) -> anyhow::Result<(Vec<usize>, Vec<f32>)> {
    let k = keep.iter().filter(|&&b| b).count();
    match kind {
        FfnKind::Gate | FfnKind::Up => {
            let (inter, hidden) = (shape[0], shape[1]);
            if keep.len() != inter {
                anyhow::bail!("defrag: keep len {} != gate/up rows {inter}", keep.len());
            }
            let mut out = Vec::with_capacity(k * hidden);
            for r in 0..inter {
                if keep[r] {
                    out.extend_from_slice(&vals[r * hidden..(r + 1) * hidden]);
                }
            }
            Ok((vec![k, hidden], out))
        }
        FfnKind::Down => {
            let (hidden, inter) = (shape[0], shape[1]);
            if keep.len() != inter {
                anyhow::bail!("defrag: keep len {} != down cols {inter}", keep.len());
            }
            let mut out = Vec::with_capacity(hidden * k);
            for r in 0..hidden {
                for c in 0..inter {
                    if keep[c] {
                        out.push(vals[r * inter + c]);
                    }
                }
            }
            Ok((vec![hidden, k], out))
        }
    }
}

/// Effective f32 for a canonical tensor: the baked overlay if present,
/// otherwise the backbone tensor from the safetensors files.
fn effective_tensor(
    overlay: &HashMap<String, (Vec<usize>, Vec<f32>)>,
    files: &[SafeTensors],
    name: &str,
) -> anyhow::Result<(Vec<usize>, Vec<f32>)> {
    if let Some((s, v)) = overlay.get(name) {
        return Ok((s.clone(), v.clone()));
    }
    for f in files {
        for m in &f.tensors {
            if canon_name(&m.name).as_deref() == Some(name) {
                return Ok((m.shape.clone(), to_f32(&m.dtype, f.bytes(m))?));
            }
        }
    }
    anyhow::bail!("defrag: tensor '{name}' not in overlay or base model")
}

struct DefragPlan {
    /// Baked FFN replacements (canonical name → shape + f32), overriding
    /// the backbone before pruning (carries FCD-retrained weights).
    overlay: HashMap<String, (Vec<usize>, Vec<f32>)>,
    /// Per-layer live-neuron mask over the intermediate dim.
    keep: HashMap<usize, Vec<bool>>,
}

/// Build the defrag plan from a skill dir: baked overlays (`tensors/*.npy`)
/// and a keep-set — explicit `ffn_keep.npy` if present, else autodetected
/// from zeroed down_proj columns (the Factory-Hard bake).
fn build_defrag_plan(
    dir: &Path,
    arch: &ModelArch,
    files: &[SafeTensors],
) -> anyhow::Result<DefragPlan> {
    let mut overlay: HashMap<String, (Vec<usize>, Vec<f32>)> = HashMap::new();
    let tdir = dir.join("tensors");
    if tdir.is_dir() {
        for entry in fs::read_dir(&tdir)? {
            let p = entry?.path();
            if p.extension().and_then(|e| e.to_str()) != Some("npy") {
                continue;
            }
            let stem = p
                .file_stem()
                .and_then(|s| s.to_str())
                .unwrap_or_default()
                .to_string();
            let a = npy::read(&p)?;
            let vals = match a.data {
                npy::NpyData::F32(v) => v,
                npy::NpyData::Bool(_) => {
                    anyhow::bail!("defrag overlay {stem}: expected float, got bool")
                }
            };
            overlay.insert(stem, (a.shape, vals));
        }
    }
    println!(
        "  Defrag overlay: {} baked tensors from {}",
        overlay.len(),
        dir.display()
    );

    let (nl, inter) = (arch.num_layers, arch.intermediate_size);
    let mut keep: HashMap<usize, Vec<bool>> = HashMap::new();
    let keep_path = dir.join("ffn_keep.npy");
    if keep_path.exists() {
        let a = npy::read(&keep_path)?;
        if a.shape != [nl, inter] {
            anyhow::bail!("ffn_keep.npy shape {:?} != model ({nl}, {inter})", a.shape);
        }
        let flags: Vec<bool> = match a.data {
            npy::NpyData::Bool(v) => v,
            npy::NpyData::F32(v) => v.iter().map(|&x| x != 0.0).collect(),
        };
        for li in 0..nl {
            let row = flags[li * inter..(li + 1) * inter].to_vec();
            if !row.iter().any(|&b| b) {
                anyhow::bail!("defrag: layer {li} has 0 live neurons");
            }
            keep.insert(li, row);
        }
    } else {
        // Producer-free: a neuron is dead iff its down_proj INPUT column is
        // all-zero (Factory-Hard bake). Reads each layer's effective down.
        println!("  Defrag: no ffn_keep.npy — autodetecting from zero down_proj columns");
        for li in 0..nl {
            let name = format!("model.layers.{li}.mlp.down_proj.weight");
            let (shape, vals) = effective_tensor(&overlay, files, &name)?;
            let (hidden, cols) = (shape[0], shape[1]);
            let mut alive = vec![false; cols];
            for r in 0..hidden {
                for c in 0..cols {
                    if vals[r * cols + c] != 0.0 {
                        alive[c] = true;
                    }
                }
            }
            if !alive.iter().any(|&b| b) {
                anyhow::bail!("defrag: layer {li} autodetected 0 live neurons");
            }
            keep.insert(li, alive);
        }
    }
    Ok(DefragPlan { overlay, keep })
}

/// Conservative record count for the streaming writer's fixed head gap.
/// A packed `[experts, out, in]` source tensor becomes one CMF directory
/// record per expert; flat checkpoints merely over-reserve here.
fn directory_record_estimate(source_tensors: usize, arch: &ModelArch) -> usize {
    let expanded_expert_records = arch.moe.as_ref().map_or(0usize, |m| {
        arch.num_layers
            .saturating_mul(m.num_experts)
            .saturating_mul(3)
    });
    source_tensors
        .saturating_add(expanded_expert_records)
        .max(4096)
}

#[cfg_attr(not(test), allow(dead_code))] // the CLI calls run_convert_towers
pub fn run_convert(
    model: &str,
    quant: &str,
    output: &str,
    hf_token: Option<&str>,
    defrag: Option<&str>,
    // O(1) Nyström runtime hint (`--o1`): recorded in header provenance,
    // weights untouched — the runtime resolves it at load (loader.rs).
    o1_hint: Option<serde_json::Value>,
    // Continue a conversion that died: the payloads already in `output` are
    // kept, and every source shard the manifest marks done is skipped —
    // download included, which is where the hours are.
    resume: bool,
    progress: impl FnMut(f32),
) -> anyhow::Result<()> {
    let outputs = vec![(quant.to_string(), output.to_string())];
    run_convert_multi(model, &outputs, hf_token, defrag, o1_hint, resume, progress)
}

/// [`run_convert`] with the MiMo-V2 tower mode (`--mimo-towers`).
#[allow(clippy::too_many_arguments)]
pub fn run_convert_towers(
    model: &str,
    quant: &str,
    output: &str,
    hf_token: Option<&str>,
    defrag: Option<&str>,
    o1_hint: Option<serde_json::Value>,
    resume: bool,
    towers: MimoTowers,
    progress: impl FnMut(f32),
) -> anyhow::Result<()> {
    let outputs = vec![(quant.to_string(), output.to_string())];
    run_convert_multi_towers(
        model, &outputs, hf_token, defrag, o1_hint, resume, towers, progress,
    )
}

/// Convert one source checkpoint into one or more independently finalized
/// CMF outputs.  All requested profiles consume each source shard in one
/// streamed pass; the decoded values are borrowed while each profile's
/// encoder writes its own payload.
#[cfg_attr(not(test), allow(dead_code))]
pub fn run_convert_multi(
    model: &str,
    outputs: &[(String, String)],
    hf_token: Option<&str>,
    defrag: Option<&str>,
    o1_hint: Option<serde_json::Value>,
    resume: bool,
    progress: impl FnMut(f32),
) -> anyhow::Result<()> {
    run_convert_multi_towers(
        model,
        outputs,
        hf_token,
        defrag,
        o1_hint,
        resume,
        MimoTowers::TextOnly,
        progress,
    )
}

/// [`run_convert_multi`] with the MiMo-V2 tower mode. `MmOnly` writes the
/// companion (no text tensors are read); `Multimodal` adds the towers, the
/// audio tokenizer encoder and the config blobs to each text output.
#[allow(clippy::too_many_arguments)]
#[cfg_attr(not(test), allow(dead_code))] // the CLI calls run_convert_multi_excluding
pub fn run_convert_multi_towers(
    model: &str,
    outputs: &[(String, String)],
    hf_token: Option<&str>,
    defrag: Option<&str>,
    o1_hint: Option<serde_json::Value>,
    resume: bool,
    towers: MimoTowers,
    progress: impl FnMut(f32),
) -> anyhow::Result<()> {
    run_convert_multi_excluding(
        model,
        outputs,
        hf_token,
        defrag,
        o1_hint,
        resume,
        towers,
        &[],
        progress,
    )
}

/// [`run_convert_multi_towers`] leaving out every tensor whose canonical
/// name matches one of `exclude` (`cortiq convert --exclude-tensors`, see
/// [`TensorExclusion`]) — in every output, on every source path of the walk
/// (resident local shards, local-ready streaming, Hugging Face streaming,
/// resume). The packers that do not go through the walk (the MiMo-V2
/// `mm-only` companion, EmbeddingGemma 2) refuse a non-empty `exclude`.
#[allow(clippy::too_many_arguments)]
pub fn run_convert_multi_excluding(
    model: &str,
    outputs: &[(String, String)],
    hf_token: Option<&str>,
    defrag: Option<&str>,
    o1_hint: Option<serde_json::Value>,
    resume: bool,
    towers: MimoTowers,
    exclude: &[String],
    mut progress: impl FnMut(f32),
) -> anyhow::Result<()> {
    anyhow::ensure!(!outputs.is_empty(), "at least one output is required");
    let exclusion = TensorExclusion::new(exclude);
    if towers == MimoTowers::MmOnly {
        anyhow::ensure!(
            defrag.is_none() && o1_hint.is_none() && !resume && exclusion.is_empty(),
            "--mimo-towers mm-only writes only the towers: --defrag, --o1, --resume and \
             --exclude-tensors do not apply"
        );
        return crate::mimo_towers::run_convert_mimo_mm(model, outputs, progress);
    }
    // EmbeddingGemma 2 is an encoder with its own packer (text + towers)
    // and its own profiles (bf16 | f32 | q8_2f | q4tp).
    if let Some(dir) = Some(Path::new(model)).filter(|d| {
        fs::read(d.join("config.json"))
            .ok()
            .and_then(|b| serde_json::from_slice::<serde_json::Value>(&b).ok())
            .is_some_and(|c| c.get("model_type").and_then(|v| v.as_str()) == Some("embedding_gemma2"))
    }) {
        anyhow::ensure!(
            defrag.is_none()
                && o1_hint.is_none()
                && !resume
                && towers == MimoTowers::TextOnly
                && exclusion.is_empty(),
            "embedding_gemma2: --defrag, --o1, --resume, --mimo-towers and --exclude-tensors do not apply"
        );
        for (q, path) in outputs {
            crate::egemma2pack::convert(dir, q, path)?;
        }
        progress(1.0);
        return Ok(());
    }
    for (i, (_, path)) in outputs.iter().enumerate() {
        anyhow::ensure!(!path.trim().is_empty(), "output path {} is empty", i + 1);
        anyhow::ensure!(
            outputs[..i].iter().all(|(_, p)| p != path),
            "output path '{}' is repeated",
            path
        );
    }
    // `auto` resolves once the config is read (default_quant_for_arch);
    // every explicit profile is validated now, before any download.
    let requested_profiles: Vec<Option<Quant>> = outputs
        .iter()
        .map(|(q, _)| {
            if q.trim().eq_ignore_ascii_case(AUTO_QUANT) {
                Ok(None)
            } else {
                parse_quant(q).map(Some)
            }
        })
        .collect::<anyhow::Result<_>>()?;

    // Source: a local HF directory, or an HF repo id — hub checkpoints
    // convert STREAMED: one weight shard on disk at a time.
    let downloaded;
    let mut stream_shards: Vec<String> = Vec::new();
    let mut stream_repo: Option<String> = None;
    let local_ready_dir = local_ready_dir_from_env();
    let dir: &Path = if Path::new(model).join("config.json").exists() {
        Path::new(model)
    } else if looks_like_repo(model) {
        anyhow::ensure!(
            local_ready_dir.is_none(),
            "{LOCAL_READY_DIR_ENV} requires a local model directory; unset it for HF hub streaming"
        );
        eprintln!("downloading {model} from Hugging Face (streamed)…");
        let (d, shards) = hf_download_opts(model, hf_token, false)?;
        stream_shards = shards;
        stream_repo = Some(model.to_string());
        downloaded = d;
        downloaded.as_path()
    } else {
        anyhow::bail!(
            "'{model}': not a local model dir (no config.json) and not an HF repo id (owner/name)"
        );
    };

    let config: serde_json::Value = serde_json::from_slice(
        &fs::read(dir.join("config.json")).map_err(|e| anyhow::anyhow!("read config.json: {e}"))?,
    )?;
    // EmbeddingGemma 2 converts from a local dir (dispatched above).
    anyhow::ensure!(
        config.get("model_type").and_then(|v| v.as_str()) != Some("embedding_gemma2"),
        "embedding_gemma2 converts from a local checkpoint dir: \
         `hf download {model} --local-dir DIR`, then `cortiq convert --model DIR`"
    );
    let mut arch = build_arch(&config)?;
    // The multimodal towers (checked before any shard is touched: a missing
    // audio tokenizer must not surface after a multi-hour text pass).
    let tower_source = match towers {
        MimoTowers::TextOnly => None,
        MimoTowers::MmOnly => unreachable!("dispatched above"),
        MimoTowers::Multimodal => {
            anyhow::ensure!(
                arch.arch_name == MIMO_V2,
                "--mimo-towers multimodal applies to {MIMO_V2} checkpoints, not '{}'",
                arch.arch_name
            );
            anyhow::ensure!(
                stream_repo.is_none(),
                "--mimo-towers multimodal needs a local checkpoint dir (with {}/)",
                crate::mimo_towers::AUDIO_TOKENIZER_DIR
            );
            Some(crate::mimo_towers::TowerSource::open(dir)?)
        }
    };
    let profiles: Vec<Quant> = requested_profiles
        .iter()
        .map(|q| {
            q.unwrap_or_else(|| {
                let d = default_quant_for_arch(&arch);
                eprintln!(
                    "  quant: none requested — {} converts to {} by default",
                    arch.arch_name,
                    quant_name(d)
                );
                d
            })
        })
        .collect();
    let quant = profiles[0];
    // Some codecs intentionally share a physical dtype (q1/q1p both use
    // Q1).  Preserve the requested encoder so later skill grafts can encode
    // replacement tensors with the same algorithm instead of guessing from
    // the directory dtype alone.
    let requested_quant = quant_name(quant);
    arch.prism_hadamard = load_prism_hadamard(dir, &config)?;
    if arch.prism_hadamard.is_some() {
        let has_ordinary = profiles.iter().any(|q| matches!(q, Quant::Q2TiledP));
        let has_affine = profiles.iter().any(|q| matches!(q, Quant::Q2TiledPAffine));
        eprintln!(
            "  Prism/Bonsai: signed FWHT-1024 enabled (profiles: q2tp={} q2tp_affine={})",
            has_ordinary, has_affine
        );
        anyhow::ensure!(
            profiles
                .iter()
                .all(|q| matches!(q, Quant::Q2TiledP | Quant::Q2TiledPAffine)),
            "Prism/Bonsai conversion supports only q2tp and q2tp_affine profiles; other profiles would discard the signed FWHT contract"
        );
    }
    let source_fp8_block = source_fp8_block(&config);
    // NVIDIA ModelOpt NVFP4 checkpoints decode to f32 first (then any
    // profile, mxfp4 included, re-encodes them).
    let nvfp4_src = nvfp4_source(dir, &config)?;
    if let Some(g) = nvfp4_src {
        eprintln!("  source: NVIDIA ModelOpt NVFP4 (E2M1 + E4M3 scale per {g} + per-tensor scale)");
    }
    if !exclusion.is_empty() {
        eprintln!(
            "  --exclude-tensors: leaving out canonical names matching {}",
            exclude.join(", ")
        );
        // The index is on disk for Hub streaming (fetched before the walk)
        // and for a sharded local checkpoint.
        let index_names: Vec<String> = fs::read(dir.join("model.safetensors.index.json"))
            .ok()
            .and_then(|b| serde_json::from_slice::<serde_json::Value>(&b).ok())
            .and_then(|v| {
                v.get("weight_map").and_then(|m| m.as_object()).map(|m| {
                    m.keys()
                        .filter_map(|k| canon_name_for_arch(&arch, k, towers))
                        .collect()
                })
            })
            .unwrap_or_default();
        exclusion.preview(&index_names);
    }
    let consume_source_shards =
        stream_repo.is_none() && arch.deepseek_v41.is_some() && consume_source_shards_enabled();
    if consume_source_shards {
        eprintln!(
            "  source consumption enabled via {CONSUME_SOURCE_SHARDS_ENV}: each local shard is deleted after its output checkpoint"
        );
    }

    // Read output checkpoints before opening local source shards. A consumed
    // shard may no longer exist, but its manifest mark is proof that every
    // output already wrote it. Only the intersection across all outputs is
    // safe to filter; a shard missing from any active output remains required.
    let mut resume_marks: Vec<std::collections::HashSet<String>> =
        vec![std::collections::HashSet::new(); outputs.len()];
    let mut resume_completed = vec![false; outputs.len()];
    if resume {
        for (i, (_, output_path)) in outputs.iter().enumerate() {
            let output = std::path::Path::new(output_path);
            if !output.exists() {
                continue;
            }
            resume_completed[i] = cortiq_core::format::CmfModel::open(output_path).is_ok();
            let manifest_path = format!("{output_path}.manifest");
            if std::path::Path::new(&manifest_path).exists() {
                resume_marks[i] = resume_manifest_marks(std::path::Path::new(&manifest_path));
            }
        }
    }
    let source_shards = if stream_repo.is_none() {
        source_shard_counts(dir)?
    } else {
        Vec::new()
    };
    if local_ready_dir.is_some() {
        anyhow::ensure!(
            stream_repo.is_none(),
            "{LOCAL_READY_DIR_ENV} cannot be combined with HF hub streaming"
        );
        anyhow::ensure!(
            !source_shards.is_empty(),
            "{LOCAL_READY_DIR_ENV} requires at least one local source shard"
        );
        stream_shards = source_shards.iter().map(|(name, _)| name.clone()).collect();
    }
    let local_ready_manifest = if let Some(ready_dir) = local_ready_dir.as_deref() {
        let required: Vec<String> = source_shards.iter().map(|(name, _)| name.clone()).collect();
        let manifest_path = local_ready_manifest_path(ready_dir);
        Some(read_source_file_manifest(&manifest_path, &required)?)
    } else {
        None
    };
    // Both the HF index and the local-ready certificate have been validated
    // before this point.  Reorder only the streamed list so resume marks and
    // the resident local conversion path retain their existing semantics.
    if !stream_shards.is_empty() {
        apply_source_shard_priority(&mut stream_shards)?;
    }
    let completed_source_shards: std::collections::HashSet<String> = if resume {
        source_shards
            .iter()
            .filter(|(name, _)| {
                outputs
                    .iter()
                    .enumerate()
                    .all(|(i, _)| resume_completed[i] || resume_marks[i].contains(name))
            })
            .map(|(name, _)| name.clone())
            .collect()
    } else {
        std::collections::HashSet::new()
    };
    let resumed_tensor_count: usize = source_shards
        .iter()
        .filter(|(name, _)| completed_source_shards.contains(name))
        .map(|(_, count)| *count)
        .sum();

    // Memory-map the weights and process one tensor at a time — the raw model is
    // never fully loaded into RAM (peak ≈ the .cmf output + one tensor).
    let streamed_source = stream_repo.is_some() || local_ready_dir.is_some();
    let mut files = if streamed_source {
        Vec::new() // shards arrive one at a time below
    } else {
        open_model_filtered(dir, &completed_source_shards)?
    };
    if streamed_source && defrag.is_some() {
        anyhow::bail!("--defrag needs a resident local checkpoint dir (streamed convert)");
    }

    // Physical defragmentation plan (spec §11): drop pruned FFN neurons so
    // they are neither stored nor computed. arch.intermediate_size becomes
    // nominal/max (per-layer truth lives in the reduced tensor shapes).
    let orig_inter = arch.intermediate_size;
    let defrag_plan = match defrag {
        Some(d) => Some(build_defrag_plan(Path::new(d), &arch, &files)?),
        None => None,
    };
    if let Some(plan) = &defrag_plan {
        let max_kept = (0..arch.num_layers)
            .filter_map(|li| plan.keep.get(&li).map(|k| k.iter().filter(|&&b| b).count()))
            .max()
            .unwrap_or(orig_inter);
        arch.intermediate_size = max_kept;
    }
    let total: usize = if stream_repo.is_some() {
        1
    } else {
        source_shards
            .iter()
            .map(|(_, count)| *count)
            .sum::<usize>()
            .max(1)
    };
    // A 300B-class MoE encodes to ~100 GB, and holding that in
    // `Vec<TensorSpec>` until the writer runs OOMs any machine (measured:
    // +1.8 GB/min, a 176 GB box exhausted mid-model). Each encoded tensor
    // therefore goes straight into the output file and is dropped; the head
    // is patched into a reserved gap once the last tensor lands, so the
    // payloads are never held twice — not in RAM and not on disk.
    // Hub streaming has no resident `files`, so `total` is one here. Packed
    // MoE tensors can nevertheless expand into E individual gate/up/down
    // directory records per layer (Qwen3.8 Flash: >73k records). Account for
    // that expansion up front; discovering an undersized head only in
    // `finish()` would otherwise waste the entire multi-hour conversion.
    // Flat-expert checkpoints are deliberately counted twice by this upper
    // bound — a larger zero-filled gap is cheap and keeps resume portable.
    let directory_records = directory_record_estimate(total, &arch);
    let head_reserve = cortiq_core::format::CmfStreamWriter::head_reserve_for(
        2usize.saturating_mul(directory_records),
        96,
    );
    let mut manifests = Vec::with_capacity(outputs.len());
    let mut done_shards: Vec<std::collections::HashSet<String>> = Vec::with_capacity(outputs.len());
    let mut writers: Vec<Option<cortiq_core::format::CmfStreamWriter>> =
        Vec::with_capacity(outputs.len());
    let mut completed = vec![false; outputs.len()];
    for (_, output_path) in outputs {
        let manifest_path = format!("{output_path}.manifest");
        // A valid CMF alongside a stale manifest means a previous multi-output
        // run finished this profile before another profile failed. Treat it as
        // complete and never append a second directory to the finalized file.
        if resume && resume_completed[manifests.len()] {
            completed[manifests.len()] = true;
            manifests.push(manifest_path);
            done_shards.push(std::collections::HashSet::new());
            writers.push(None);
            continue;
        }
        let (writer, marks) = if resume
            && std::path::Path::new(&manifest_path).exists()
            && std::path::Path::new(output_path).exists()
        {
            let (w, st) = cortiq_core::format::CmfStreamWriter::resume(output_path, &manifest_path)
                .map_err(|e| anyhow::anyhow!("resume {output_path}: {e}"))?;
            check_resume_exclusion(output_path, &manifest_path, exclude, &exclusion, &st.names)?;
            tracing::info!(
                "resuming {output_path}: {} tensors already written, {} source shards done",
                st.names.len(),
                st.marks.len()
            );
            let w = w
                .appending_manifest(&manifest_path)
                .map_err(|e| anyhow::anyhow!("resume {output_path}: {e}"))?;
            (Some(w), st.marks.into_iter().collect())
        } else {
            let w = cortiq_core::format::CmfStreamWriter::new(output_path, head_reserve)
                .and_then(|w| w.with_manifest(&manifest_path))
                .map_err(|e| anyhow::anyhow!("create {output_path}: {e}"))?;
            record_exclusion(&manifest_path, exclude)
                .map_err(|e| anyhow::anyhow!("create {output_path}: {e}"))?;
            (Some(w), std::collections::HashSet::new())
        };
        manifests.push(manifest_path);
        done_shards.push(marks);
        writers.push(writer);
    }
    // Source shards earlier attempts finished (for the exclusion report).
    let shards_done_before = done_shards
        .iter()
        .map(|d| d.len())
        .max()
        .unwrap_or(0)
        .max(completed_source_shards.len());
    let mut tensors: Vec<Vec<TensorSpec>> =
        outputs.iter().map(|_| Vec::with_capacity(total)).collect();
    let mut tower_stats = vec![crate::mimo_towers::TowerStats::default(); outputs.len()];
    if arch.prism_hadamard.is_some() {
        let mut vision_cfg = config
            .get("vision_config")
            .cloned()
            .ok_or_else(|| anyhow::anyhow!("Prism checkpoint has no vision_config"))?;
        // qwen3vis accepts this optional list; an empty list is the honest
        // value for this source (no deepstack merger tensors are present).
        if let Some(obj) = vision_cfg.as_object_mut() {
            obj.entry("deepstack_visual_indexes")
                .or_insert_with(|| serde_json::json!([]));
        }
        let cfg_bytes = serde_json::to_vec(&vision_cfg)?;
        for (i, batch) in tensors.iter_mut().enumerate() {
            if !completed[i] {
                batch.push(TensorSpec {
                    name: "vis.config_json".into(),
                    dtype: TensorDtype::U8,
                    shape: vec![cfg_bytes.len()],
                    data: cfg_bytes.clone(),
                });
            }
        }
    }
    // Count source tensors whose durable shard marks let us skip opening the
    // (possibly already consumed) input. This keeps progress monotonic across
    // a crash while the remaining files are processed below.
    let mut done = resumed_tensor_count;
    if done > 0 {
        progress(done as f32 / total as f32);
    }
    // Tiny cross-shard tensors (gemma-4 router.scale, ~128 f32 each):
    // stashed as shards stream by, so a projection in shard 2 can fold
    // a scale that lived in the already-deleted shard 1.
    let mut small_stash: std::collections::HashMap<String, Vec<f32>> =
        std::collections::HashMap::new();
    // Per-file conversion body, shared by the resident local path and
    // the streaming hub path (shard downloaded → converted → DELETED;
    // peak disk = one shard + the growing output).
    // MiniCPM residual-branch scale (folded at write).
    let resid_fold: Option<f32> = if arch.arch_name == "minicpm3" {
        let tcv = config.get("text_config").unwrap_or(&config);
        let sd = tcv
            .get("scale_depth")
            .and_then(|v| v.as_f64())
            .unwrap_or(1.0);
        Some((sd / (arch.num_layers as f64).sqrt()) as f32)
    } else {
        None
    };
    // Kimi: which layer indices are KDA (canon retag inside the loop).
    // Kimi: which layer indices are KDA (canon retag inside the loop).
    let kda_layers: std::collections::HashSet<usize> = arch
        .layer_types
        .iter()
        .enumerate()
        .filter(|(_, t)| matches!(t, cortiq_core::LayerType::Kda))
        .map(|(i, _)| i)
        .collect();
    // The source affine scales are fit to a 32-group ladder once per matrix.
    // Keep the measured fit error for the immutable conversion report/header
    // instead of presenting the rung quantization as lossless.
    let mut prism_affine_scale_sq = 0.0f64;
    let mut prism_affine_scale_count = 0usize;
    // MiMo-V2 folds attention_value_scale into the V rows it splits out of
    // the fused qkv.
    let mimo_value_scale = if arch.arch_name == MIMO_V2 {
        mimo_v2_value_scale(config.get("text_config").unwrap_or(&config))?
    } else {
        1.0
    };
    let convert_only = match std::env::var(CONVERT_ONLY_ENV) {
        Ok(spec) => {
            let pats = parse_convert_only(&spec)?;
            eprintln!(
                "  {CONVERT_ONLY_ENV}={spec}: writing a PARTIAL file (only matching tensors) — a probe, not a model"
            );
            Some(pats)
        }
        Err(_) => None,
    };

    let mut process_file = |file: &SafeTensors,
                            files: &[SafeTensors],
                            batches: &mut [Vec<TensorSpec>],
                            profiles: &[Quant],
                            active: &[bool],
                            writers: &mut [Option<cortiq_core::format::CmfStreamWriter>],
                            done: &mut usize,
                            total: usize,
                            progress: &mut dyn FnMut(f32)|
     -> anyhow::Result<()> {
        for m in &file.tensors {
            if m.name.ends_with(".router.scale") {
                small_stash.insert(m.name.clone(), to_f32(&m.dtype, file.bytes(m))?);
            }
        }
        for m in &file.tensors {
            *done += 1;
            progress(*done as f32 / total as f32);
            if let Some(prism) = arch.prism_hadamard.as_ref() {
                if m.name.ends_with(".signs") {
                    anyhow::ensure!(
                        m.dtype == "F32" && m.shape.len() == 1,
                        "Prism sign tensor {} must be F32 [width], got {} {:?}",
                        m.name,
                        m.dtype,
                        m.shape
                    );
                    let expected = prism.signs_for_width(m.shape[0]).ok_or_else(|| {
                        anyhow::anyhow!(
                            "Prism sign tensor {} has unsupported width {}",
                            m.name,
                            m.shape[0]
                        )
                    })?;
                    let actual = to_f32(&m.dtype, file.bytes(m))?;
                    anyhow::ensure!(
                        actual.len() == expected.len()
                            && actual
                                .iter()
                                .zip(expected)
                                .all(|(a, b)| a.to_bits() == b.to_bits()),
                        "Prism sign tensor {} does not match hadamard.json",
                        m.name
                    );
                    continue;
                }
            }
            let Some(name) = canon_name_for_arch(&arch, &m.name, towers) else {
                continue;
            };
            if let Some(pats) = convert_only.as_ref() {
                if !pats.iter().any(|p| glob_match(p, &name)) {
                    continue;
                }
            }
            // MiMo-V2 towers (multimodal mode only — text mode dropped them
            // in canon): the tower codec policy, not the text profile rules.
            if arch.arch_name == MIMO_V2
                && cortiq_engine::mimo_mm::MimoTowerGroup::of(&name).is_some()
            {
                if exclusion.skip_source(&name, file.bytes(m).len()) {
                    continue;
                }
                crate::mimo_towers::emit_tower(
                    batches,
                    profiles,
                    active,
                    &mut tower_stats,
                    &name,
                    &m.shape,
                    &m.dtype,
                    file.bytes(m),
                )?;
                continue;
            }
            // HunYuan dense ships the tied `lm_head.weight` as a second copy
            // of the embedding; the loader reads the embedding when the
            // header says tied, so the copy would only be dead bytes.
            if arch.arch_name == "hunyuan_v1_dense"
                && arch.tie_word_embeddings
                && name == "lm_head.weight"
            {
                tracing::info!("hunyuan_v1_dense: tied lm_head.weight not written (embedding serves)");
                continue;
            }
            if arch.qwen4_exp.is_some()
                && ((name.starts_with("model.mtp.") && !qwen4_mtp_sidecar())
                    || name.ends_with("ple_embedding.layer_multipliers")
                    || name.ends_with("ple_embedding.ngram_heads_offsets")
                    || name.ends_with("ple_embedding.ngram_heads_vocab_sizes"))
            {
                // qwen4 MTP is speculative-only and uses a different stack; it
                // goes to its own sidecar file (`--mtp-sidecar`), never into
                // the main file. PLE's integer hash tables are deterministic
                // from the header and are recomputed by the runtime, avoiding
                // lossy casts.
                continue;
            }
            // Kimi: KDA layers share the `self_attn.` vendor prefix with
            // the MLA full-attention layers — retag them by the layer
            // schedule so the loader dispatches unambiguously.
            let name = if kda_layers.is_empty() {
                name
            } else {
                match name
                    .strip_prefix("model.layers.")
                    .and_then(|r| r.split_once('.'))
                    .and_then(|(li, rest)| Some((li.parse::<usize>().ok()?, rest)))
                {
                    Some((li, rest))
                        if kda_layers.contains(&li) && rest.starts_with("self_attn.") =>
                    {
                        format!("model.layers.{li}.kda_attn.{}", &rest["self_attn.".len()..])
                    }
                    _ => name,
                }
            };
            // `--exclude-tensors`: the canonical name is final here (a split
            // below is checked again at the writer). Nothing has read the
            // payload yet — the slice length is mmap metadata.
            if exclusion.skip_source(&name, file.bytes(m).len()) {
                continue;
            }

            // DeepSeek-V4.1 Engram tables are native FP8 byte planes. Keep
            // both planes lossless as explicit U8 auxiliary tensors before
            // any generic scale skipping or FP8 decode can consume them.
            // Their companion shape is checked in the source shard (the
            // published checkpoint co-locates each pair).
            let raw_engram =
                m.name.ends_with(".engram.embed.weight") || m.name.ends_with(".engram.embed.scale");
            if arch.deepseek_v41.is_some() && raw_engram {
                let is_weight = m.name.ends_with(".engram.embed.weight");
                anyhow::ensure!(m.shape.len() == 2, "{name}: Engram table must be 2-D");
                anyhow::ensure!(
                    m.shape.iter().all(|&d| d > 0),
                    "{name}: Engram table has an empty dimension"
                );
                let expected_dtype = if is_weight { "F8_E4M3" } else { "F8_E8M0" };
                anyhow::ensure!(
                    m.dtype == expected_dtype,
                    "{name}: Engram {} has dtype {}, expected {expected_dtype}",
                    if is_weight { "weight" } else { "scale" },
                    m.dtype
                );
                if is_weight {
                    anyhow::ensure!(
                        m.shape[1] % 32 == 0,
                        "{name}: Engram weight columns {} are not divisible by 32",
                        m.shape[1]
                    );
                }
                let stem = if is_weight {
                    m.name
                        .strip_suffix(".weight")
                        .expect("Engram weight suffix checked")
                } else {
                    m.name
                        .strip_suffix(".scale")
                        .expect("Engram scale suffix checked")
                };
                let companion_name = if is_weight {
                    format!("{stem}.scale")
                } else {
                    format!("{stem}.weight")
                };
                let companion = files.iter().find_map(|source| {
                    source
                        .tensors
                        .iter()
                        .find(|t| t.name == companion_name)
                        .map(|t| (t.dtype.clone(), t.shape.clone()))
                });
                let (comp_dtype, comp_shape) = companion.ok_or_else(|| {
                    anyhow::anyhow!("{name}: missing Engram companion '{companion_name}'")
                })?;
                if is_weight {
                    let expected_scale_cols = m.shape[1].div_ceil(32);
                    anyhow::ensure!(
                        comp_dtype == "F8_E8M0"
                            && comp_shape == vec![m.shape[0], expected_scale_cols],
                        "{name}: Engram scale companion has dtype {comp_dtype} shape {:?}, expected F8_E8M0 [{}, {}]",
                        comp_shape,
                        m.shape[0],
                        expected_scale_cols
                    );
                } else {
                    let expected_weight_cols = m.shape[1]
                        .checked_mul(32)
                        .ok_or_else(|| anyhow::anyhow!("{name}: Engram shape overflow"))?;
                    anyhow::ensure!(
                        comp_dtype == "F8_E4M3"
                            && comp_shape == vec![m.shape[0], expected_weight_cols],
                        "{name}: Engram weight companion has dtype {comp_dtype} shape {:?}, expected F8_E4M3 [{}, {}]",
                        comp_shape,
                        m.shape[0],
                        expected_weight_cols
                    );
                }
                push_raw_u8_tensor(writers, active, &name, &m.shape, file.bytes(m))?;
                continue;
            }

            // Skip MLX scales and biases as they are processed with the weight.
            if m.dtype == "F16" && (name.ends_with(".scales") || name.ends_with(".biases")) {
                continue;
            }
            // MXFP4 group scales ride with their .weight_packed (Kimi-K3) or
            // U8 .weight (MiMo-V2) twin.
            if m.dtype == "U8" && name.ends_with(".weight_scale") {
                continue;
            }
            // DeepSeek-V4/V4.1 keeps every quantized tensor's E8M0 scale plane in
            // a sibling `.scale`; it rides with the weight below.
            if m.dtype == "F8_E8M0" && name.ends_with(".scale") {
                continue;
            }
            // Fine-grained FP8 scale planes ride with `.weight` below.
            if name.ends_with(".weight_scale_inv") {
                continue;
            }
            // NVFP4 (ModelOpt): block and global weight scales ride with
            // their `.weight`; activation and KV-cache calibration scalars
            // and the AWQ pre-quant scale (folded below) are not weights.
            if nvfp4_src.is_some()
                && [
                    ".weight_scale",
                    ".weight_scale_2",
                    ".input_scale",
                    ".k_scale",
                    ".v_scale",
                    ".pre_quant_scale",
                ]
                .iter()
                .any(|s| m.name.ends_with(s))
            {
                continue;
            }
            // MiMo-V2 fused qkv: TP-chunked rows AND a TP-chunked scale grid.
            // It must be decoded here, before the generic FP8 path: that
            // path tiles the scales continuously, which refuses the
            // full-attention layers (108 vs 106 scale rows) but silently
            // ACCEPTS the sliding layers (116 = 116), and the Phi-3 split
            // below would then read global [q|k|v] out of chunked rows.
            if arch.arch_name == MIMO_V2 && name.ends_with(".self_attn.qkv_proj.weight") {
                let li = layer_index_of(&name)
                    .ok_or_else(|| anyhow::anyhow!("{name}: no layer index"))?;
                anyhow::ensure!(
                    m.dtype == "F8_E4M3" && m.shape.len() == 2,
                    "{name}: expected a 2-D F8_E4M3 fused qkv, got {} {:?}",
                    m.dtype,
                    m.shape
                );
                let scale_name = format!("{}_scale_inv", m.name);
                let (scale_vals, scale_shape) = files
                    .iter()
                    .find_map(|source| {
                        source
                            .tensors
                            .iter()
                            .find(|t| t.name == scale_name)
                            .map(|t| (source, t))
                    })
                    .map(|(source, t)| {
                        anyhow::ensure!(
                            t.dtype == "F32",
                            "{scale_name}: expected F32 inverse scales, got {}",
                            t.dtype
                        );
                        Ok((to_f32(&t.dtype, source.bytes(t))?, t.shape.clone()))
                    })
                    .ok_or_else(|| anyhow::anyhow!("{name}: missing {scale_name}"))??;
                let n_kv_heads = arch
                    .kv_heads_per_layer
                    .as_ref()
                    .and_then(|k| k.get(li).copied())
                    .ok_or_else(|| anyhow::anyhow!("{name}: no KV head count for layer {li}"))?;
                let layout = MimoQkvLayout {
                    n_heads: arch.num_attention_heads,
                    n_kv_heads,
                    head_dim: arch.head_dim,
                    v_head_dim: arch.v_head_dim.unwrap_or(arch.head_dim),
                    ckpt_tp: arch.num_kv_heads,
                };
                let (rows, cols) = (m.shape[0], m.shape[1]);
                let (q, k, v) = mimo_v2_split_qkv(
                    file.bytes(m),
                    &scale_vals,
                    &scale_shape,
                    rows,
                    cols,
                    layout,
                    source_fp8_block,
                    mimo_value_scale,
                )
                .map_err(|e| anyhow::anyhow!("{name}: {e}"))?;
                for (part, vals) in [("q_proj", &q), ("k_proj", &k), ("v_proj", &v)] {
                    emit_profiled_tensor(
                        batches,
                        profiles,
                        active,
                        &arch,
                        &name.replace("qkv_proj", part),
                        &[vals.len() / cols, cols],
                        vals,
                        EmitMode::Auto,
                    );
                }
                continue;
            }
            let fp8_scale_inv = if m.dtype == "F8_E4M3" && m.shape.len() == 2 {
                let scale_name = m.name.replace(".weight", ".weight_scale_inv");
                let mut scale = None;
                for source in files {
                    if let Some(sm) = source.tensors.iter().find(|t| t.name == scale_name) {
                        scale = Some(to_f32(&sm.dtype, source.bytes(sm))?);
                        break;
                    }
                }
                match scale {
                    Some(scales) => Some((
                        m.shape.clone(),
                        unpack_fp8_scale_inv(
                            file.bytes(m),
                            &scales,
                            m.shape[0],
                            m.shape[1],
                            source_fp8_block,
                        )?,
                    )),
                    None => None,
                }
            } else {
                None
            };
            // `fp4` profile / `mxfp4` tensor quant: an OCP MXFP4 source
            // (DeepSeek-V4/V4.1 experts: I8 [rows, cols/2] nibble pairs +
            // E8M0 [rows, cols/32] scales) goes into the file as its own two
            // planes — no dequant, no requant. Outputs that want another
            // codec still take the f32 decode below.
            let raw_fp4_rest: Vec<bool>;
            let active: &[bool] = match dsv4_raw_mxfp4(
                &arch,
                &name,
                m,
                file,
                files,
                profiles,
                active,
                defrag_plan.is_none(),
            )? {
                Some((payload, shape, took)) => {
                    for ((batch, &t), &a) in batches.iter_mut().zip(&took).zip(active) {
                        if t && a {
                            batch.push(TensorSpec {
                                name: name.clone(),
                                dtype: TensorDtype::Mxfp4,
                                shape: shape.clone(),
                                data: payload.clone(),
                            });
                        }
                    }
                    raw_fp4_rest = active.iter().zip(&took).map(|(&a, &t)| a && !t).collect();
                    if !raw_fp4_rest.iter().any(|&a| a) {
                        continue;
                    }
                    &raw_fp4_rest
                }
                None => active,
            };
            // Its own two quantizations, decoded to f32 so the rest of the
            // pipeline (defrag, requant to q4tp/q2tp) is layout-agnostic:
            //   * experts  — I8 holding two FP4 (E2M1) values per byte with
            //     one E8M0 scale per 32 values: OCP MXFP4 exactly.
            //   * skeleton — F8_E4M3 with one E8M0 scale per source tile
            //     (128x128 for V4, 32x32 for V4.1).
            let dsv4 = if matches!(m.dtype.as_str(), "I8" | "F8_E4M3") && !name.ends_with(".scale")
            {
                let scale_name = format!("{}.scale", m.name.trim_end_matches(".weight"));
                let mut sb = None;
                for f in files {
                    if let Some(t) = f.tensors.iter().find(|t| t.name == scale_name) {
                        sb = Some(f.bytes(t));
                    }
                }
                match sb {
                    Some(scales) if m.shape.len() == 2 => {
                        let (rows, cols) = (m.shape[0], m.shape[1]);
                        if m.dtype == "I8" {
                            // packed FP4: `cols` bytes = 2*cols values
                            Some((
                                vec![rows, cols * 2],
                                unpack_mxfp4(file.bytes(m), scales, rows, cols)?,
                            ))
                        } else {
                            Some((
                                vec![rows, cols],
                                unpack_fp8_blocks(
                                    file.bytes(m),
                                    scales,
                                    rows,
                                    cols,
                                    source_fp8_block,
                                )?,
                            ))
                        }
                    }
                    _ => None,
                }
            } else {
                None
            };

            let mxfp4_packed = m.dtype == "U8" && m.name.ends_with(".weight_packed");
            let mxfp4 = if mxfp4_packed {
                // MXFP4 (Kimi-K3 experts): decode to f32 and continue the
                // normal path under the plain `.weight` name.
                let scale_name = m.name.replace(".weight_packed", ".weight_scale");
                let mut scales_blob = None;
                for f in files {
                    if let Some(t) = f.tensors.iter().find(|t| t.name == scale_name) {
                        scales_blob = Some(f.bytes(t));
                    }
                }
                let scales = scales_blob
                    .ok_or_else(|| anyhow::anyhow!("missing {scale_name} for mxfp4 unpacking"))?;
                anyhow::ensure!(m.shape.len() == 2, "{name}: mxfp4 expects 2-D");
                let (rows, cp) = (m.shape[0], m.shape[1]);
                Some((
                    vec![rows, cp * 2],
                    unpack_mxfp4(file.bytes(m), scales, rows, cp)?,
                ))
            } else if nvfp4_src.is_some() && m.dtype == "U8" && m.name.ends_with(".weight") {
                // NVIDIA ModelOpt NVFP4: decode with the E4M3 block scales and
                // the per-tensor `weight_scale_2`, fold an AWQ pre-quant scale
                // into the columns, then continue as an ordinary f32 tensor.
                Some(nvfp4_decode(m, file, files)?)
            } else if m.dtype == "U8" && m.name.ends_with(".weight") {
                // MXFP4 under the plain `.weight` name (MiMo-V2 experts): U8
                // [rows, cols/2] nibble pairs + a U8 E8M0 sibling
                // `.weight_scale` [rows, cols/32]. Without that sibling the
                // bytes are some other packing, and the generic path below
                // refuses the U8 dtype as before.
                let scale_name = format!("{}_scale", m.name);
                let scale = files.iter().find_map(|f| {
                    f.tensors
                        .iter()
                        .find(|t| t.name == scale_name)
                        .map(|t| (f.bytes(t), t.dtype.as_str(), t.shape.clone()))
                });
                match scale {
                    Some((scales, scale_dtype, scale_shape)) => {
                        anyhow::ensure!(m.shape.len() == 2, "{name}: mxfp4 expects 2-D");
                        let (rows, cp) = (m.shape[0], m.shape[1]);
                        anyhow::ensure!(
                            scale_dtype == "U8"
                                && (cp * 2) % 32 == 0
                                && scale_shape == vec![rows, cp * 2 / 32],
                            "{name}: U8 weight [{rows}, {cp}] with {scale_name} {scale_dtype} {:?} \
                             is not MXFP4 (expected U8 [{rows}, {}])",
                            scale_shape,
                            cp * 2 / 32
                        );
                        Some((
                            vec![rows, cp * 2],
                            unpack_mxfp4(file.bytes(m), scales, rows, cp)?,
                        ))
                    }
                    None => None,
                }
            } else {
                None
            };
            let name = if mxfp4_packed {
                name.strip_suffix("_packed")
                    .expect("suffix checked")
                    .to_string()
            } else {
                name
            };

            // Prism source matrices are MLX U32 bitplanes.  The production
            // q2tp_affine profile preserves the source 0/1/2 symbols in the
            // existing dtype16 plane, while ordinary q2tp uses the existing
            // midrise encoder.  Both outputs can be produced in one streamed
            // pass over the source shard.
            if arch.prism_hadamard.is_some() && m.dtype == "U32" && m.name.ends_with(".weight") {
                anyhow::ensure!(
                    m.shape.len() == 2,
                    "Prism packed matrix {name} must be 2-D, got {:?}",
                    m.shape
                );
                let out_dim = m.shape[0];
                let in_dim = m.shape[1]
                    .checked_mul(16)
                    .ok_or_else(|| anyhow::anyhow!("Prism matrix {name} input width overflows"))?;
                let find_companion = |suffix: &str| {
                    let wanted = m.name.replace(".weight", suffix);
                    files.iter().find_map(|source| {
                        source
                            .tensors
                            .iter()
                            .find(|t| t.name == wanted)
                            .map(|t| (source.bytes(t), t.dtype.as_str(), t.shape.clone()))
                    })
                };
                let (scales, scale_dtype, scale_shape) =
                    find_companion(".scales").ok_or_else(|| {
                        anyhow::anyhow!("missing {}.scales for Prism matrix {name}", m.name)
                    })?;
                let (biases, bias_dtype, bias_shape) =
                    find_companion(".biases").ok_or_else(|| {
                        anyhow::anyhow!("missing {}.biases for Prism matrix {name}", m.name)
                    })?;
                anyhow::ensure!(
                    scale_dtype == "F16"
                        && bias_dtype == "F16"
                        && scale_shape == vec![out_dim, in_dim / 128]
                        && bias_shape == scale_shape,
                    "Prism affine companions for {name} have dtypes/shapes ({scale_dtype} {:?}, {bias_dtype} {:?}), expected F16 [{out_dim}, {}]",
                    scale_shape,
                    bias_shape,
                    in_dim / 128
                );
                let descriptor = arch.prism_hadamard.as_ref().unwrap();
                anyhow::ensure!(
                    descriptor.signs_for_width(in_dim).is_some(),
                    "Prism matrix {name} input width {in_dim} has no signed FWHT vector"
                );
                validate_mlx_ternary_codes(file.bytes(m), out_dim, in_dim)?;

                let affine_payload = if profiles.iter().any(|q| matches!(q, Quant::Q2TiledPAffine))
                {
                    let (payload, fit_rms) =
                        encode_mlx_q2tp_affine(file.bytes(m), scales, biases, out_dim, in_dim)?;
                    anyhow::ensure!(
                        fit_rms.is_finite(),
                        "Prism matrix {name} produced a non-finite affine scale fit"
                    );
                    prism_affine_scale_sq +=
                        (fit_rms as f64).powi(2) * (out_dim * (in_dim / GROUP_SIZE)) as f64;
                    prism_affine_scale_count = prism_affine_scale_count
                        .saturating_add(out_dim.saturating_mul(in_dim / GROUP_SIZE));
                    Some(payload)
                } else {
                    None
                };

                // Only the ordinary profile needs a dense source view.  It is
                // bounded to one matrix and is immediately dropped after
                // both output batches receive their independent payloads.
                let ordinary_vals = if profiles.iter().any(|q| matches!(q, Quant::Q2TiledP)) {
                    Some(unpack_mlx(
                        file.bytes(m),
                        scales,
                        Some(biases),
                        out_dim,
                        in_dim,
                        2,
                    )?)
                } else {
                    None
                };

                for ((batch, requested), &is_active) in batches.iter_mut().zip(profiles).zip(active)
                {
                    if !is_active {
                        continue;
                    }
                    match *requested {
                        Quant::Q2TiledP => {
                            emit_profiled_tensor(
                                std::slice::from_mut(batch),
                                std::slice::from_ref(requested),
                                std::slice::from_ref(&is_active),
                                &arch,
                                &name,
                                &[out_dim, in_dim],
                                ordinary_vals
                                    .as_ref()
                                    .expect("ordinary q2tp source decode built above"),
                                EmitMode::Auto,
                            );
                        }
                        Quant::Q2TiledPAffine => {
                            batch.push(TensorSpec {
                                name: name.clone(),
                                dtype: TensorDtype::Q2TiledP,
                                shape: vec![out_dim, in_dim],
                                data: affine_payload
                                    .as_ref()
                                    .expect("q2tp affine payload built above")
                                    .clone(),
                            });
                        }
                        _ => unreachable!("Prism profile gate above"),
                    }
                }
                continue;
            }
            let mxfp4 = fp8_scale_inv.or(mxfp4).or(dsv4);
            // MLX Conv3d stores Qwen's patch kernel as [out, temporal, height,
            // width, channels], while the CMF vision tower consumes flattened
            // per-patch rows in the source processor's [channels, temporal,
            // height, width] order.  Keep this tower lossless, but make the
            // shape/axis contract explicit so VisionTower::from_cmf receives a
            // normal 2-D projection instead of an unusable 5-D safetensor.
            let prism_patch = arch.prism_hadamard.is_some() && name == "vis.patch_embed.weight";
            let (m_shape, m_vals) = if prism_patch {
                flatten_prism_patch_weight(&m.dtype, &m.shape, file.bytes(m))?
            } else if let Some(v) = mxfp4 {
                v
            } else if m.dtype == "U32" && m.name.ends_with(".weight") {
                let scales_name = m.name.replace(".weight", ".scales");
                let biases_name = m.name.replace(".weight", ".biases");
                let mut scales_blob = None;
                let mut biases_blob = None;
                for f in files {
                    if let Some(t) = f.tensors.iter().find(|t| t.name == scales_name) {
                        scales_blob = Some(f.bytes(t));
                    }
                    if let Some(t) = f.tensors.iter().find(|t| t.name == biases_name) {
                        biases_blob = Some(f.bytes(t));
                    }
                }
                let scales = scales_blob
                    .ok_or_else(|| anyhow::anyhow!("missing {} for MLX unpacking", scales_name))?;
                let out_dim = m.shape[0];
                let w_cols = m.shape[1];
                let num_groups = scales.len() / 2 / out_dim;

                let mut bits = 0;
                let mut in_dim = 0;
                for b in [1, 2, 3, 4, 8] {
                    let possible_in_dim = w_cols * 32 / b;
                    if possible_in_dim % num_groups == 0 {
                        let gs = possible_in_dim / num_groups;
                        if gs == 32 || gs == 64 || gs == 128 {
                            bits = b;
                            in_dim = possible_in_dim;
                            break;
                        }
                    }
                }
                if bits == 0 {
                    anyhow::bail!(
                        "Could not deduce MLX bit width for shape {:?} and {} scale groups",
                        m.shape,
                        num_groups
                    );
                }
                (
                    vec![out_dim, in_dim],
                    unpack_mlx(file.bytes(m), scales, biases_blob, out_dim, in_dim, bits)?,
                )
            } else {
                (m.shape.clone(), to_f32(&m.dtype, file.bytes(m))?)
            };

            let mut m_vals = m_vals;
            // MiniCPM: every residual add is h += branch·(scale_depth/√L)
            // — fold the factor into the branch output rows (o_proj and
            // the dense FFN down_proj) so the runtime stays standard.
            if let Some(fs) = resid_fold {
                if name.ends_with("self_attn.o_proj.weight")
                    || name.ends_with(".mlp.down_proj.weight")
                {
                    for v in m_vals.iter_mut() {
                        *v *= fs;
                    }
                }
            }

            // DeepSeek-V2 MLA: permute each q head to rope-first
            // ([nope|rope] → [rope|nope]) so the runtime's standard
            // partial rotary (first rotary_dim dims) covers it.
            if arch.mla.is_some()
                && (name.ends_with("self_attn.q_proj.weight")
                    || name.ends_with("self_attn.q_b_proj.weight"))
            {
                let mla = arch.mla.as_ref().unwrap();
                let (dr, dn) = (mla.qk_rope_head_dim, mla.qk_nope_head_dim);
                let hd = dr + dn;
                anyhow::ensure!(
                    m_shape.len() == 2 && m_shape[0] % hd == 0,
                    "{name}: rows {:?} not a multiple of qk head dim {hd}",
                    m_shape
                );
                let cols = m_shape[1];
                let nh = m_shape[0] / hd;
                let mut w = vec![0.0f32; m_vals.len()];
                for h in 0..nh {
                    for r in 0..dr {
                        // DeepSeek rotary interleaves pairs (view d/2,2 →
                        // transpose): store rope dims even-first so our
                        // half-split rotation reproduces their math.
                        // Kimi NoPE (no rotation) and MiniCPM3 (plain
                        // half-split rotate_half) keep the rope block
                        // order — only the move in front of nope.
                        let interleaved = !mla.nope && arch.arch_name.contains("deepseek");
                        let src = if !interleaved {
                            r
                        } else if r < dr / 2 {
                            2 * r
                        } else {
                            2 * (r - dr / 2) + 1
                        };
                        w[(h * hd + r) * cols..(h * hd + r + 1) * cols].copy_from_slice(
                            &m_vals[(h * hd + dn + src) * cols..(h * hd + dn + src + 1) * cols],
                        );
                    }
                    for r in 0..dn {
                        w[(h * hd + dr + r) * cols..(h * hd + dr + r + 1) * cols]
                            .copy_from_slice(&m_vals[(h * hd + r) * cols..(h * hd + r + 1) * cols]);
                    }
                }
                emit_profiled_tensor(
                    batches,
                    &profiles,
                    active,
                    &arch,
                    &name,
                    &m_shape,
                    &w,
                    EmitMode::Auto,
                );
                continue;
            }

            // DeepSeek MLA: the shared rope key rows (tail of kv_a) get
            // the same even-first interleave fix. Kimi NoPE skips it —
            // no rotation, the tail rows are already where the loader
            // expects them.
            if arch.mla.as_ref().is_some_and(|m| !m.nope)
                && arch.arch_name.contains("deepseek")
                && name.ends_with("self_attn.kv_a_proj_with_mqa.weight")
            {
                let mla = arch.mla.as_ref().unwrap();
                let (lora, dr) = (mla.kv_lora_rank, mla.qk_rope_head_dim);
                anyhow::ensure!(
                    m_shape.len() == 2 && m_shape[0] == lora + dr,
                    "{name}: rows {:?} != lora+rope",
                    m_shape
                );
                let cols = m_shape[1];
                let mut w = m_vals.clone();
                for r in 0..dr {
                    let src = if r < dr / 2 {
                        2 * r
                    } else {
                        2 * (r - dr / 2) + 1
                    };
                    w[(lora + r) * cols..(lora + r + 1) * cols]
                        .copy_from_slice(&m_vals[(lora + src) * cols..(lora + src + 1) * cols]);
                }
                emit_profiled_tensor(
                    batches,
                    &profiles,
                    active,
                    &arch,
                    &name,
                    &m_shape,
                    &w,
                    EmitMode::Auto,
                );
                continue;
            }

            // Gemma-4 MoE: packed 3-D expert tensors + a router whose
            // input gain (scale-less rms ⊙ router.scale · hidden^-1/2)
            // folds entirely into the projection columns. Experts split
            // into the canonical per-expert 2-D matrices; the fused
            // gate_up halves are [gate | up] along the last axis.
            if name.ends_with(".experts.gate_up_proj") || name.ends_with(".experts.down_proj") {
                anyhow::ensure!(
                    m_shape.len() == 3,
                    "{name}: expected 3-D, got {:?}",
                    m_shape
                );
                let base = name
                    .strip_suffix(".experts.gate_up_proj")
                    .or_else(|| name.strip_suffix(".experts.down_proj"))
                    .unwrap();
                // Two lineages name the packed tensor differently: gemma-4
                // has `layers.N.experts.*`, qwen3.5/3.6 `layers.N.mlp.experts.*`.
                // The output name below appends `.mlp.experts`, so a base that
                // already ends in `.mlp` would double it — and a doubled prefix
                // converts silently, then fails at load with "router present but
                // no expert tensors", which points nowhere near here.
                let base = base.strip_suffix(".mlp").unwrap_or(base);
                let moe = arch
                    .moe
                    .as_ref()
                    .ok_or_else(|| anyhow::anyhow!("no moe cfg"))?;
                let (mi, hid) = (moe.moe_intermediate_size, arch.hidden_size);
                let (ne, d1, d2) = (m_shape[0], m_shape[1], m_shape[2]);
                let is_gu = name.ends_with("gate_up_proj");
                // Both orientations occur in the wild: standard per-expert
                // nn.Linear [out, in] rows (gemma-4 QAT) or the packed
                // [in, out] (Mixtral lineage). Detect by dims.
                let row_major = if is_gu {
                    if (d1, d2) == (2 * mi, hid) {
                        true
                    } else if (d1, d2) == (hid, 2 * mi) {
                        false
                    } else {
                        anyhow::bail!("{name}: dims {:?} match neither orientation", m_shape)
                    }
                } else if (d1, d2) == (hid, mi) {
                    true
                } else if (d1, d2) == (mi, hid) {
                    false
                } else {
                    anyhow::bail!("{name}: dims {:?} match neither orientation", m_shape)
                };
                for e in 0..ne {
                    let ev = &m_vals[e * d1 * d2..(e + 1) * d1 * d2];
                    if is_gu {
                        let mut gate = vec![0.0f32; mi * hid];
                        let mut up = vec![0.0f32; mi * hid];
                        if row_major {
                            gate.copy_from_slice(&ev[..mi * hid]);
                            up.copy_from_slice(&ev[mi * hid..]);
                        } else {
                            for h in 0..hid {
                                for r in 0..mi {
                                    gate[r * hid + h] = ev[h * d2 + r];
                                    up[r * hid + h] = ev[h * d2 + mi + r];
                                }
                            }
                        }
                        let gate_name = format!("{base}.mlp.experts.{e}.gate_proj.weight");
                        emit_profiled_tensor(
                            batches,
                            &profiles,
                            active,
                            &arch,
                            &gate_name,
                            &[mi, hid],
                            &gate,
                            EmitMode::Auto,
                        );
                        let up_name = format!("{base}.mlp.experts.{e}.up_proj.weight");
                        emit_profiled_tensor(
                            batches,
                            &profiles,
                            active,
                            &arch,
                            &up_name,
                            &[mi, hid],
                            &up,
                            EmitMode::Auto,
                        );
                    } else {
                        let mut down = vec![0.0f32; hid * mi];
                        if row_major {
                            down.copy_from_slice(ev);
                        } else {
                            for r in 0..mi {
                                for h in 0..hid {
                                    down[h * mi + r] = ev[r * d2 + h];
                                }
                            }
                        }
                        let down_name = format!("{base}.mlp.experts.{e}.down_proj.weight");
                        emit_profiled_tensor(
                            batches,
                            &profiles,
                            active,
                            &arch,
                            &down_name,
                            &[hid, mi],
                            &down,
                            EmitMode::Auto,
                        );
                    }
                }
                continue;
            }
            if name.ends_with(".router.proj.weight") {
                anyhow::ensure!(
                    m_shape.len() == 2,
                    "{name}: expected 2-D, got {:?}",
                    m_shape
                );
                let (ne, hid) = (m_shape[0], m_shape[1]);
                // Fold router.scale ⊙ hidden^-1/2 into the columns.
                let scale_name = m.name.replace(".proj.weight", ".scale");
                let mut scale_blob = small_stash.get(&scale_name).cloned();
                if scale_blob.is_none() {
                    for f in files {
                        if let Some(t) = f.tensors.iter().find(|t| t.name == scale_name) {
                            scale_blob = Some(to_f32(&t.dtype, f.bytes(t))?);
                        }
                    }
                }
                let scale =
                    scale_blob.ok_or_else(|| anyhow::anyhow!("missing {scale_name} for fold"))?;
                anyhow::ensure!(scale.len() == hid, "{scale_name}: len != hidden");
                let c = 1.0 / (hid as f32).sqrt();
                let mut w = m_vals.clone();
                for r in 0..ne {
                    for j in 0..hid {
                        w[r * hid + j] *= scale[j] * c;
                    }
                }
                let base = name.strip_suffix(".router.proj.weight").unwrap();
                let out_name = format!("{base}.mlp.gate.weight");
                emit_profiled_tensor(
                    batches,
                    &profiles,
                    active,
                    &arch,
                    &out_name,
                    &[ne, hid],
                    &w,
                    EmitMode::F16,
                );
                continue;
            }
            if name.ends_with(".router.scale") {
                continue; // folded into the router projection above
            }
            if name.ends_with(".router.per_expert_scale") {
                let base = name.strip_suffix(".router.per_expert_scale").unwrap();
                let out_name = format!("{base}.mlp.per_expert_scale");
                emit_profiled_tensor(
                    batches,
                    &profiles,
                    active,
                    &arch,
                    &out_name,
                    &[m_vals.len()],
                    &m_vals,
                    EmitMode::F16,
                );
                continue;
            }

            // qwen3_next / AgentWorld fuse the GDN projections (in_proj_qkvz /
            // in_proj_ba) with a group-interleaved layout; split them natively
            // into the canonical hub tensors (in_proj_qkv/z/a/b). Pure row
            // permutation — no value is changed.
            if name.contains(".linear_attn.in_proj_qkvz")
                || name.contains(".linear_attn.in_proj_ba")
            {
                if m_shape.len() != 2 {
                    anyhow::bail!("fused GDN tensor '{name}': expected 2-D, got {:?}", m_shape);
                }
                let w = &m_vals;
                let hid = m_shape[1];
                let miss = |k: &str| anyhow::anyhow!("fused GDN needs {k} in config");
                let nk = arch
                    .linear_num_key_heads
                    .ok_or_else(|| miss("linear_num_key_heads"))?;
                let dk = arch
                    .linear_key_head_dim
                    .ok_or_else(|| miss("linear_key_head_dim"))?;
                let nv = arch
                    .linear_num_value_heads
                    .ok_or_else(|| miss("linear_num_value_heads"))?;
                let dv = arch
                    .linear_value_head_dim
                    .ok_or_else(|| miss("linear_value_head_dim"))?;
                for (out_name, out_vals, out_rows) in
                    split_fused_gdn(&name, w, hid, nk, dk, nv, dv)?
                {
                    emit_profiled_tensor(
                        batches,
                        &profiles,
                        active,
                        &arch,
                        &out_name,
                        &[out_rows, hid],
                        &out_vals,
                        EmitMode::Auto,
                    );
                }
                continue;
            }
            // Phi-3 family fuses QKV (`qkv_proj`) and gate/up
            // (`gate_up_proj`): split into the canonical tensors — a
            // pure row slice, no value changes.
            if name.ends_with(".self_attn.qkv_proj.weight")
                || name.ends_with(".mlp.gate_up_proj.weight")
            {
                anyhow::ensure!(m_shape.len() == 2, "fused '{name}': expected 2-D");
                let w = &m_vals;
                let (rows, cols) = (m_shape[0], m_shape[1]);
                let parts: Vec<(String, usize, usize)> = if name.contains("qkv_proj") {
                    let q = arch.num_attention_heads * arch.head_dim;
                    let kv = arch.num_kv_heads * arch.head_dim;
                    anyhow::ensure!(
                        q + 2 * kv == rows,
                        "'{name}': {rows} rows != q({q}) + 2·kv({kv})"
                    );
                    vec![
                        (name.replace("qkv_proj", "q_proj"), 0, q),
                        (name.replace("qkv_proj", "k_proj"), q, kv),
                        (name.replace("qkv_proj", "v_proj"), q + kv, kv),
                    ]
                } else {
                    anyhow::ensure!(rows % 2 == 0, "'{name}': odd row count {rows}");
                    vec![
                        (name.replace("gate_up_proj", "gate_proj"), 0, rows / 2),
                        (name.replace("gate_up_proj", "up_proj"), rows / 2, rows / 2),
                    ]
                };
                for (out_name, r0, nr) in parts {
                    let vals = &w[r0 * cols..(r0 + nr) * cols];
                    emit_profiled_tensor(
                        batches,
                        &profiles,
                        active,
                        &arch,
                        &out_name,
                        &[nr, cols],
                        vals,
                        EmitMode::Auto,
                    );
                }
                continue;
            }
            // Gemma-4 global (full-attention) layers carry no v_proj —
            // V is the K projection, normalized separately at runtime
            // (attention_k_eq_v). Materialize the duplicate so the
            // runtime keeps its uniform Q/K/V/O contract; the overlay
            // costs one MQA-sized tensor per global layer.
            if arch.global_head_dim.is_some() && name.ends_with(".self_attn.k_proj.weight") {
                let li: Option<usize> = name
                    .split(".layers.")
                    .nth(1)
                    .and_then(|r| r.split('.').next())
                    .and_then(|n| n.parse().ok());
                let pat = arch.sliding_window_pattern.unwrap_or(usize::MAX);
                if let Some(li) = li {
                    if (li + 1) % pat == 0 {
                        anyhow::ensure!(m_shape.len() == 2, "'{name}': expected 2-D");
                        let w = &m_vals;
                        let (rows, cols) = (m_shape[0], m_shape[1]);
                        for out_name in [name.clone(), name.replace("k_proj", "v_proj")] {
                            emit_profiled_tensor(
                                batches,
                                &profiles,
                                active,
                                &arch,
                                &out_name,
                                &[rows, cols],
                                w,
                                EmitMode::Auto,
                            );
                        }
                        continue;
                    }
                }
            }
            // Defrag: for an FFN weight of a pruned layer, take the baked
            // overlay (if any) else the backbone value, drop dead neurons
            // (gate/up rows, down columns), then quantize the reduced shape.
            // The neuron never enters the blob — nor the runtime's math.
            if let Some(plan) = defrag_plan.as_ref() {
                if let Some((li, kind)) = ffn_kind(&name) {
                    if let Some(keep) = plan.keep.get(&li) {
                        let (shape, vals) = match plan.overlay.get(&name) {
                            Some((s, v)) => (s.clone(), v.clone()),
                            None => (m_shape.clone(), m_vals.clone()),
                        };
                        let (out_shape, out_vals) = slice_ffn(&kind, &shape, &vals, keep)?;
                        emit_profiled_tensor(
                            batches,
                            &profiles,
                            active,
                            &arch,
                            &name,
                            &out_shape,
                            &out_vals,
                            EmitMode::Auto,
                        );
                        continue;
                    }
                }
            }
            let vals = m_vals;
            let numel: usize = m_shape.iter().product();
            if numel != vals.len() {
                anyhow::bail!(
                    "tensor '{name}': {} values for shape {:?}",
                    vals.len(),
                    m_shape
                );
            }
            // 1-D tensors, tiny tensors, non-2-D, and gate-critical projections go f16.
            // Index tables ride as raw f32: quantizing them would round
            // expert ids, and f16 cannot even hold a vocabulary id exactly.
            if force_f32(&name) {
                emit_profiled_tensor(
                    batches,
                    &profiles,
                    active,
                    &arch,
                    &name,
                    &m_shape,
                    &vals,
                    EmitMode::F32,
                );
                continue;
            }
            emit_profiled_tensor(
                batches,
                &profiles,
                active,
                &arch,
                &name,
                &m_shape,
                &vals,
                EmitMode::Auto,
            );
        }

        Ok(())
    };
    if stream_repo.is_some() || local_ready_dir.is_some() {
        let base = stream_repo
            .as_deref()
            .map(|repo| format!("https://huggingface.co/{repo}/resolve/main"));
        let threads = hf_threads();
        let agent = ureq::AgentBuilder::new()
            .timeout_connect(Duration::from_secs(20))
            .timeout_read(Duration::from_secs(300))
            .build();
        let ns = stream_shards.len().max(1);
        for (si, sname) in stream_shards.iter().enumerate() {
            let active: Vec<bool> = outputs
                .iter()
                .enumerate()
                .map(|(i, _)| !completed[i] && !done_shards[i].contains(sname))
                .collect();
            if !active.iter().any(|&v| v) {
                eprintln!(
                    "  [stream {}/{}] {sname} — все выходы уже в файле, пропуск",
                    si + 1,
                    ns
                );
                progress((si as f32 + 1.0) / ns as f32);
                continue;
            }
            eprintln!("  [stream {}/{}] {sname}", si + 1, ns);
            let source_path = if let Some(base) = base.as_deref() {
                fetch(
                    &agent,
                    &format!("{base}/{sname}"),
                    &dir.join(sname),
                    hf_token,
                    true,
                    threads,
                )?;
                dir.join(sname)
            } else {
                let ready_dir = local_ready_dir.as_deref().ok_or_else(|| {
                    anyhow::anyhow!("streamed source has neither an HF repo nor a ready directory")
                })?;
                let expected = local_ready_manifest
                    .as_ref()
                    .and_then(|m| m.get(sname))
                    .ok_or_else(|| {
                        anyhow::anyhow!(
                            "local-ready source manifest has no expectation for shard {sname}"
                        )
                    })?;
                wait_for_local_ready_source(ready_dir, dir, sname, expected)?
            };
            let f = open_safetensors(&source_path)?;
            let one = [f];
            let ft = one[0].tensors.len().max(1);
            let mut fd = 0usize;
            let mut sub = |p: f32| progress((si as f32 + p) / ns as f32);
            process_file(
                &one[0],
                &one,
                &mut tensors,
                &profiles,
                &active,
                &mut writers,
                &mut fd,
                ft,
                &mut sub,
            )?;
            // Spill this shard's payloads before touching the next one:
            // that is what keeps residency at one shard instead of the
            // whole model.
            for i in 0..outputs.len() {
                if !active[i] {
                    continue;
                }
                let writer = writers[i].as_mut().ok_or_else(|| {
                    anyhow::anyhow!("output {} is already finalized", outputs[i].1)
                })?;
                drain_to_writer(&mut tensors[i], writer, &exclusion)?;
                // Only now is this shard's work durable in the output file.
                writer
                    .mark(sname)
                    .map_err(|e| anyhow::anyhow!("manifest: {e}"))?;
                done_shards[i].insert(sname.clone());
            }
            drop(one);
            if stream_repo.is_some() {
                // Preserve the existing hub-stream cleanup behavior. Hub
                // cache files are disposable and this path predates the
                // strict local-ready disk-budget contract.
                let _ = fs::remove_file(&source_path);
            } else if consume_source_shards {
                // Local-ready consumption is the bounded-disk path: a failed
                // unlink must stop conversion instead of silently retaining
                // another 100 GB shard and overrunning the pod volume.
                consume_local_ready_source(&source_path)?;
            }
        }
    } else {
        for file_index in 0..files.len() {
            let file_name = files[file_index].name.clone();
            let file_tensor_count = files[file_index].tensors.len();
            let active: Vec<bool> = outputs
                .iter()
                .enumerate()
                .map(|(i, _)| !completed[i] && !done_shards[i].contains(&file_name))
                .collect();
            if !active.iter().any(|&v| v) {
                eprintln!("  [{file_name}] уже в файле, пропуск");
                done += file_tensor_count;
                progress(done as f32 / total as f32);
                if consume_source_shards {
                    consume_source_shard(&mut files, file_index, dir)?;
                }
                continue;
            }
            process_file(
                &files[file_index],
                &files,
                &mut tensors,
                &profiles,
                &active,
                &mut writers,
                &mut done,
                total,
                &mut progress,
            )?;
            // Marking per source file gives a local conversion the same
            // resumability as a streamed one; the payloads have to reach the
            // output first, so the drain comes before the mark.
            for i in 0..outputs.len() {
                if !active[i] {
                    continue;
                }
                let writer = writers[i].as_mut().ok_or_else(|| {
                    anyhow::anyhow!("output {} is already finalized", outputs[i].1)
                })?;
                drain_to_writer(&mut tensors[i], writer, &exclusion)?;
                writer
                    .mark(&file_name)
                    .map_err(|e| anyhow::anyhow!("manifest: {e}"))?;
                done_shards[i].insert(file_name.clone());
            }
            if consume_source_shards {
                // Drop the mmap before unlinking so the filesystem can
                // reclaim this shard's blocks immediately. Every active
                // output has flushed its payload and manifest mark above.
                consume_source_shard(&mut files, file_index, dir)?;
            }
        }
    }

    // MiMo-V2 multimodal: the audio tokenizer encoder (its own file) and
    // the two config blobs, after the shard pass so the text outputs'
    // resumable shard marks stay exactly the index's shards.
    if let Some(src) = tower_source.as_ref() {
        let active: Vec<bool> = (0..outputs.len()).map(|i| !completed[i]).collect();
        src.emit_tail(&mut tensors, &profiles, &active, &mut tower_stats)?;
    }

    // Tokenizer + chat bundle (optional but recommended).
    let tok_cfg: serde_json::Value = fs::read(dir.join("tokenizer_config.json"))
        .ok()
        .and_then(|b| serde_json::from_slice(&b).ok())
        .unwrap_or(serde_json::Value::Null);
    let vocab = fs::read(dir.join("tokenizer.json")).ok().or_else(|| {
        // Kimi (tiktoken): synthesize a standard tokenizer.json from the
        // rank table so the runtime tokenizer stays unchanged.
        let tk = fs::read_to_string(dir.join("tiktoken.model")).ok()?;
        match tiktoken_to_tokenizer_json(&tk, &tok_cfg) {
            Ok(j) => {
                tracing::info!("tiktoken.model → tokenizer.json ({} bytes)", j.len());
                Some(j.into_bytes())
            }
            Err(e) => {
                eprintln!("  tiktoken.model found but conversion failed: {e}");
                None
            }
        }
    });
    let gen_cfg: serde_json::Value = fs::read(dir.join("generation_config.json"))
        .ok()
        .and_then(|b| serde_json::from_slice(&b).ok())
        .unwrap_or(serde_json::Value::Null);
    // Sidecar `chat_template.jinja` first, then the tokenizer_config field;
    // ignore an empty/blank file so we correctly fall through to the config.
    let chat_template = fs::read_to_string(dir.join("chat_template.jinja"))
        .ok()
        .filter(|s| !s.trim().is_empty())
        .or_else(|| {
            tok_cfg
                .get("chat_template")
                .and_then(|v| v.as_str().map(String::from))
        });
    let bundle = TokenizerBundle {
        chat_template,
        eos_token_ids: {
            let mut e = eos_ids(&gen_cfg, &config);
            // HunYuan hy_v3 declares a separate end-of-document id; a
            // generation that emits it is finished too.
            if let Some(eod) = config.get("eod_token_id").and_then(|v| v.as_u64()) {
                if !e.contains(&(eod as u32)) {
                    e.push(eod as u32);
                }
            }
            e
        },
        bos_token_id: config
            .get("bos_token_id")
            .and_then(|v| v.as_u64())
            .map(|n| n as u32),
        pad_token_id: config
            .get("pad_token_id")
            .and_then(|v| v.as_u64())
            .map(|n| n as u32),
    };

    let quant_type = |q: Quant| match q {
        Quant::Q8Row => QuantType::Q8Row,
        Quant::Q8_2f => QuantType::Q8_2f,
        Quant::Q4Block => QuantType::Q4Block,
        Quant::F16 => QuantType::F16,
        Quant::Vbit => QuantType::Vbit,
        Quant::Q4Tiled
        | Quant::Q4TiledP
        | Quant::Q2TiledP
        | Quant::Q2TiledPAffine
        | Quant::Mxfp4
        | Quant::Fp4 => QuantType::Q4Block,
        // File-level label only (per-tensor truth is in the directory);
        // Vbit is the closest existing informational bucket for q1.
        Quant::Q1 | Quant::Q1p | Quant::Q1s | Quant::Q1t => QuantType::Vbit,
    };
    let provenance = match &defrag_plan {
        Some(plan) => {
            let kept: Vec<usize> = (0..arch.num_layers)
                .map(|li| {
                    plan.keep
                        .get(&li)
                        .map(|k| k.iter().filter(|&&b| b).count())
                        .unwrap_or(orig_inter)
                })
                .collect();
            let live: usize = kept.iter().sum();
            let ratio = 1.0 - live as f64 / (arch.num_layers as f64 * orig_inter as f64);
            eprintln!(
                "defrag: FFN pruned per-layer, {live}/{} live ({:.0}% pruned), inter {orig_inter} -> max {} (per-layer variable); masks dropped",
                arch.num_layers * orig_inter,
                ratio * 100.0,
                arch.intermediate_size
            );
            serde_json::json!({
                "tool": "cortiq convert",
                "source_model": model,
                "weight_quant": requested_quant,
                "defrag": {
                    "source_skill": defrag,
                    "pre_intermediate": orig_inter,
                    "post_intermediate_max": arch.intermediate_size,
                    "kept_per_layer": kept,
                    "pruned_ratio": (ratio * 10000.0).round() / 10000.0,
                }
            })
        }
        None => serde_json::json!({
            "tool": "cortiq convert",
            "source_model": model,
            "weight_quant": requested_quant,
        }),
    };
    let provenance = if arch.arch_name == "whisper" {
        let mut p = provenance;
        p["whisper_config"] = config.clone();
        if !gen_cfg.is_null() {
            p["whisper_generation_config"] = gen_cfg.clone();
        }
        p["task"] = serde_json::json!("automatic-speech-recognition");
        if let Some(repo) = config
            .get("_name_or_path")
            .and_then(|v| v.as_str())
            .filter(|s| !s.starts_with('/') && s.split('/').count() == 2)
        {
            p["source_model"] = serde_json::json!(repo);
        }
        p
    } else {
        provenance
    };
    let provenance = match o1_hint {
        Some(h) => {
            let mut p = provenance;
            p["o1_attn"] = h;
            p
        }
        None => provenance,
    };
    let provenance = match sampling_defaults() {
        Some(s) => {
            let mut p = provenance;
            p["sampling"] = s;
            p
        }
        None => provenance,
    };
    // A CMF_CONVERT_ONLY probe must never pass for a model.
    let provenance = match &convert_only {
        Some(pats) => {
            let mut p = provenance;
            p["partial_convert_only"] = serde_json::json!(pats);
            p
        }
        None => provenance,
    };
    // `--exclude-tensors`: what the file deliberately leaves out of the
    // source (checkers subtract these from the expected inventory).
    let provenance = if exclusion.is_empty() {
        provenance
    } else {
        let mut p = provenance;
        p["excluded_tensors"] = serde_json::json!(exclude);
        p
    };
    let provenance = if let Some(prism) = arch.prism_hadamard.as_ref() {
        let mut p = provenance;
        p["prism_hadamard"] = serde_json::json!({
            "version": prism.version,
            "block_size": prism.block_size,
            "transform": prism.transform.clone(),
            "sign_mode": prism.sign_mode.clone(),
            "widths": prism.widths.clone(),
            "gdn_v_grouped": prism.gdn_v_grouped,
            "activation_f16": prism.activation_f16,
            "q2tp_affine_profile": profiles
                .iter()
                .any(|q| matches!(q, Quant::Q2TiledPAffine)),
            "q2tp_affine_source_scale_ladder_rms": if prism_affine_scale_count == 0 {
                0.0
            } else {
                (prism_affine_scale_sq / prism_affine_scale_count as f64).sqrt()
            },
            "q2tp_affine_source_scale_ladder_groups": prism_affine_scale_count,
            "ordinary_q2tp_existing_codec": profiles
                .iter()
                .any(|q| matches!(q, Quant::Q2TiledP)),
            "ordinary_q2tp_profile_policy": "existing_mixed_q2tp_gate_up_q4tp_other",
            "ordinary_q2tp_weight_error_is_lossy": true,
            "vision_tensors_preserved": true,
            "vision_runtime": "qwen3vis_tensors_preserved_no_text_fusion",
        });
        p
    } else {
        provenance
    };
    let header_for = |i: usize| {
        let mut p = provenance.clone();
        p["weight_quant"] = serde_json::json!(quant_name(profiles[i]));
        if let Some(src) = tower_source.as_ref() {
            let mut mm = src.provenance(towers.label(), &tower_stats[i]);
            // A resumed run did not see the towers of shards finished
            // earlier; its codec summary covers only this run's part.
            mm["resumed_run"] = serde_json::json!(resume);
            p["mimo_mm"] = mm;
        }
        let header_arch = if matches!(profiles[i], Quant::Q2TiledPAffine) {
            let mut a = arch.clone();
            let prism = a
                .prism_hadamard
                .as_mut()
                .expect("Prism affine profile requires Hadamard metadata");
            let mut target_names = prism
                .forward_weight_names
                .iter()
                .chain(prism.inverse_weight_names.iter())
                .map(|name| {
                    name.strip_prefix("language_model.")
                        .unwrap_or(name)
                        .to_string()
                })
                .collect::<Vec<_>>();
            target_names.sort();
            target_names.dedup();
            prism.affine = Some(PrismAffineConfig {
                version: 1,
                profile: "q2tp_affine".into(),
                group_size: GROUP_SIZE,
                correction_scale: 0.5,
                target_names,
            });
            a
        } else {
            arch.clone()
        };
        CmfHeader {
            format: "cmf".into(),
            version: CMF_VERSION,
            arch: header_arch,
            quant_type: quant_type(profiles[i]),
            provenance: Some(p),
            tokenizer_config: Some(bundle.clone()),
            section_hashes: None,
            skills: Vec::new(),
            shard: None,
            calibration: None,
            routing: None,
            genome: None,
            lineage: Vec::new(),
            router: None,
            segments: Vec::new(),
        }
    };

    // Lay the blob out in EXECUTION order — embed, then each layer's tensors
    // contiguously (attention, then FFN, with MoE experts grouped per expert),
    // then final norm, lm_head, MTP, then any tail. HF safetensors are often
    // alphabetical (`layers.10` before `layers.2`), which scatters the decode
    // traversal across the file; sequential layer layout streams cold-start
    // reads at disk rate and lets a per-layer `madvise(WILLNEED)` cover one
    // contiguous range. Pure layout — the directory carries offsets, so the
    // reader (which addresses tensors by name/offset) is unaffected.
    for i in 0..outputs.len() {
        if completed[i] {
            continue;
        }
        tensors[i].sort_by(|a, b| {
            exec_order_key(&a.name)
                .cmp(&exec_order_key(&b.name))
                .then_with(|| a.name.cmp(&b.name))
        });
        // Anything still resident (small tensors produced outside the main
        // loop) is appended last.
        let writer = writers[i]
            .take()
            .ok_or_else(|| anyhow::anyhow!("output {} has no writer", outputs[i].1))?;
        let mut writer = writer;
        drain_to_writer(&mut tensors[i], &mut writer, &exclusion)?;
        writer
            .finish(&header_for(i), None, vocab.as_deref())
            .map_err(|e| anyhow::anyhow!("write {}: {e}", outputs[i].1))?;
        // The manifest only exists to rescue an interrupted run. Keep it
        // until every requested output has finalized, so a crash between two
        // finish calls can detect the already-valid sibling on resume.
    }
    exclusion.report(shards_done_before);
    for manifest_path in manifests {
        let _ = std::fs::remove_file(exclusion_record_path(&manifest_path));
        let _ = std::fs::remove_file(manifest_path);
    }
    progress(1.0);
    Ok(())
}

/// Whether this tensor is one of an expert's two SwiGLU input planes.
/// Trunk names are canonicalized to gate/up_proj; DeepSeek-V4's MTP stack
/// deliberately keeps its upstream ffn.w1/w3 schema, so both are part of
/// the same q2tp profile.
/// The dense FFN's gate/up projections (no expert in the name) — what the
/// q2tp profile puts on the 2-bit plane when the model has no experts.
pub(crate) fn q2tp_dense_gate_or_up(name: &str) -> bool {
    !name.contains(".experts.")
        && !name.contains(".shared_expert")
        // the MTP draft block stays 4-bit: it is one layer, and its
        // agreement with the trunk is what speculation lives on
        && !name.contains(".mtp.")
        && (name.ends_with(".mlp.gate_proj.weight")
            || name.ends_with(".mlp.up_proj.weight")
            || name.ends_with(".mlp.w1.weight")
            || name.ends_with(".mlp.w3.weight"))
}

pub(crate) fn q2tp_expert_gate_or_up(name: &str) -> bool {
    let expert = name.contains(".experts.")
        || name.contains(".shared_expert.")
        || name.contains(".shared_experts.");
    expert
        && (name.ends_with(".gate_proj.weight")
            || name.ends_with(".up_proj.weight")
            || name.ends_with(".w1.weight")
            || name.ends_with(".w3.weight"))
}

#[cfg(test)]
pub(crate) mod tests {
    use super::*;
    use cortiq_core::format::CmfModel;

    #[test]
    fn sampling_spec_parses_into_the_header_shape() {
        let v = parse_sampling("temperature=1.0, top_p=0.95,top_k=0,min_p=0,repetition_penalty=1.0")
            .unwrap();
        assert_eq!(
            v,
            serde_json::json!({"temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0.0,
                               "repetition_penalty": 1.0})
        );
        let c = cortiq_engine::SamplerConfig::from_provenance(Some(&serde_json::json!({"sampling": v})));
        assert_eq!((c.top_k, c.min_p, c.repetition_penalty), (0, 0.0, 1.0));
        let mt = parse_sampling("temperature=1.0,max_tokens=2048").unwrap();
        assert_eq!(mt["max_tokens"], serde_json::json!(2048));
        assert_eq!(
            cortiq_engine::sampler::file_max_tokens(Some(&serde_json::json!({"sampling": mt}))),
            Some(2048)
        );
        assert!(parse_sampling("max_tokens=-5").is_err());
        assert!(parse_sampling("top_k=-1").is_err());
        assert!(parse_sampling("temp=1").is_err());
        assert!(parse_sampling("").is_err());
    }

    #[test]
    fn whisper_config_and_native_tensor_names_are_supported() {
        let cfg = serde_json::json!({
            "model_type": "whisper", "d_model": 64,
            "encoder_layers": 2, "encoder_attention_heads": 4,
            "encoder_ffn_dim": 128, "decoder_layers": 1,
            "decoder_attention_heads": 4, "decoder_ffn_dim": 128,
            "vocab_size": 51866, "num_mel_bins": 128,
            "max_source_positions": 1500, "max_target_positions": 448,
            "layer_norm_eps": 0.00001
        });
        let arch = build_arch(&cfg).unwrap();
        assert_eq!(arch.arch_name, "whisper");
        assert_eq!(arch.hidden_size, 64);
        assert_eq!(arch.num_layers, 2);
        assert_eq!(arch.num_attention_heads, 4);
        assert_eq!(arch.vocab_size, 51866);
        assert_eq!(arch.max_position_embeddings, 1500);
        assert_eq!(arch.hidden_act, "gelu");
        for name in [
            "model.encoder.layers.0.self_attn.q_proj.weight",
            "model.encoder.layers.0.self_attn.out_proj.weight",
            "model.decoder.embed_tokens.weight",
        ] {
            assert_eq!(canon_name_for_arch(&arch, name, MimoTowers::TextOnly).as_deref(), Some(name));
        }
    }

    #[test]
    fn whisper_q4tp_keeps_attention_and_output_embedding_at_q8() {
        let cfg = serde_json::json!({
            "model_type": "whisper", "d_model": 1280,
            "encoder_layers": 32, "encoder_attention_heads": 20,
            "encoder_ffn_dim": 5120, "decoder_layers": 32,
            "decoder_attention_heads": 20, "decoder_ffn_dim": 5120,
            "vocab_size": 51866, "num_mel_bins": 128,
            "max_source_positions": 1500, "max_target_positions": 448,
            "layer_norm_eps": 0.00001
        });
        let arch = build_arch(&cfg).unwrap();
        let q4tp = Quant::Q4TiledP;
        assert_eq!(profile_quant(&arch, q4tp, "model.encoder.layers.0.self_attn.q_proj.weight"), Quant::Q8_2f);
        assert_eq!(profile_quant(&arch, q4tp, "model.decoder.layers.0.encoder_attn.v_proj.weight"), Quant::Q8_2f);
        assert_eq!(profile_quant(&arch, q4tp, "model.decoder.embed_tokens.weight"), Quant::Q8_2f);
        assert_eq!(profile_quant(&arch, q4tp, "model.encoder.layers.0.fc1.weight"), Quant::Q4TiledP);
        assert!(force_f16("model.encoder.embed_positions.weight"));
        assert!(force_f16("model.decoder.embed_positions.weight"));
    }

    #[test]
    fn deterministic_log2_exp2_track_std_to_an_ulp() {
        let ulp = |a: f32, b: f32| (a.to_bits() as i64 - b.to_bits() as i64).abs();
        let mut x = 1.0e-9f32;
        while x < 1.0e9 {
            assert!(ulp(det_log2(x), x.log2()) <= 1 || (det_log2(x) - x.log2()).abs() < 1e-7, "log2({x})");
            x *= 1.37;
        }
        let mut y = -40.0f32;
        while y < 40.0 {
            assert!(ulp(det_exp2(y), y.exp2()) <= 1, "exp2({y})");
            y += 0.173;
        }
        // exact on integers and on the special values
        assert_eq!(det_exp2(-3.0), 0.125);
        assert_eq!(det_log2(8.0), 3.0);
        assert_eq!(det_log2(0.0), f32::NEG_INFINITY);
        // the core ladder and the encoder's agree to an ulp per rung
        let params = [0x00u8, 0xbc, 0x66, 0x2e]; // lo = −1.0, step ≈ 0.1
        let (a, b) = (det_q4tp_ladder(&params, 0), cortiq_core::quant::q4tp_ladder(&params, 0));
        for c in 0..32 {
            assert!(ulp(a[c], b[c]) <= 2, "rung {c}: {} vs {}", a[c], b[c]);
        }
    }

    #[test]
    fn hf_thread_env_is_bounded_and_keeps_legacy_default() {
        assert_eq!(parse_hf_threads(None), HF_DEFAULT_THREADS);
        assert_eq!(parse_hf_threads(Some("")), HF_DEFAULT_THREADS);
        assert_eq!(parse_hf_threads(Some("not-a-number")), HF_DEFAULT_THREADS);
        assert_eq!(parse_hf_threads(Some("0")), HF_DEFAULT_THREADS);
        assert_eq!(parse_hf_threads(Some(" -4 ")), HF_DEFAULT_THREADS);
        assert_eq!(parse_hf_threads(Some(" 16 ")), 16);
        assert_eq!(parse_hf_threads(Some("64")), HF_MAX_THREADS);
        assert_eq!(parse_hf_threads(Some("65")), HF_MAX_THREADS);
        assert_eq!(
            parse_hf_threads(Some("999999999999999999999999")),
            HF_DEFAULT_THREADS
        );
    }

    #[test]
    fn hf_range_planner_covers_file_without_overlap() {
        assert!(plan_hf_ranges(0, 32).is_empty());
        assert!(plan_hf_ranges(100, 0).is_empty());

        let ranges = plan_hf_ranges(100, 32);
        assert_eq!(ranges, vec![(0, 32), (32, 64), (64, 96), (96, 100)]);
        assert_eq!(ranges.first().map(|r| r.0), Some(0));
        assert_eq!(ranges.last().map(|r| r.1), Some(100));
        for pair in ranges.windows(2) {
            assert_eq!(pair[0].1, pair[1].0, "ranges must be adjacent");
            assert!(pair[0].0 < pair[0].1, "ranges must be non-empty");
        }
        assert!(ranges.last().unwrap().0 < ranges.last().unwrap().1);
    }

    #[test]
    fn source_shard_priority_is_exact_and_stable() {
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        unsafe {
            std::env::remove_var(SOURCE_SHARD_PRIORITY_ENV);
        }
        let mut shards = vec![
            "model-00001-of-00004.safetensors".to_string(),
            "model-00002-of-00004.safetensors".to_string(),
            "model-00003-of-00004.safetensors".to_string(),
            "model-00004-of-00004.safetensors".to_string(),
        ];
        apply_source_shard_priority(&mut shards)
            .expect("no priority environment should preserve the source order");
        assert_eq!(
            shards,
            vec![
                "model-00001-of-00004.safetensors",
                "model-00002-of-00004.safetensors",
                "model-00003-of-00004.safetensors",
                "model-00004-of-00004.safetensors",
            ]
        );

        unsafe {
            std::env::set_var(
                SOURCE_SHARD_PRIORITY_ENV,
                "model-00003-of-00004.safetensors,model-00001-of-00004.safetensors",
            );
        }
        apply_source_shard_priority(&mut shards).unwrap();
        assert_eq!(
            shards,
            vec![
                "model-00003-of-00004.safetensors",
                "model-00001-of-00004.safetensors",
                "model-00002-of-00004.safetensors",
                "model-00004-of-00004.safetensors",
            ]
        );

        for bad in [
            "",
            ",model-00001-of-00004.safetensors",
            "model-00001-of-00004.safetensors,",
            "model-00099-of-00004.safetensors",
            "model-00001-of-00004.safetensors,model-00001-of-00004.safetensors",
        ] {
            unsafe {
                std::env::set_var(SOURCE_SHARD_PRIORITY_ENV, bad);
            }
            let mut candidate = shards.clone();
            assert!(
                apply_source_shard_priority(&mut candidate).is_err(),
                "priority {bad:?} must fail closed"
            );
        }
        unsafe {
            std::env::remove_var(SOURCE_SHARD_PRIORITY_ENV);
        }
    }

    /// The encoders read their switches from the environment, and the test
    /// harness runs tests on threads of ONE process — so two tests steering
    /// different switches at once make each other's results meaningless.
    /// Everything that touches those variables takes this first.
    pub(crate) static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());

    /// Pin the encoder switches for the duration of a test, whatever ran
    /// before — including the default q2tp encoder: a test that panicked
    /// while it held `CMF_Q2TP_ENCODER=v1` must not leak it into the rest.
    pub(crate) fn env_guard(threads: &str, search: &str) -> std::sync::MutexGuard<'static, ()> {
        let g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        unsafe {
            std::env::set_var("CMF_ENCODE_THREADS", threads);
            std::env::set_var("CMF_Q2TP_SEARCH", search);
            std::env::remove_var("CMF_Q2TP_ENCODER");
        }
        g
    }
    use cortiq_core::quant::{
        dequant_q4_block, dequant_q4_tiled, dequant_q4tp, dequant_q8_2f, dequant_q8_row,
        dequant_vbit, expected_nbytes,
    };

    /// Deterministic pseudo-random weights whose per-TILE amplitude spreads
    /// over `spread` powers of two within each row — that spread is the only
    /// thing a per-row scale ladder has to cope with, so it is the axis worth
    /// parameterizing. Measured on KAT-Coder-V2.5, real rows sit at a median
    /// spread of 1.27 and a 90th percentile of 1.89; 3.7 is past the tail.
    fn synth_rows(rows: usize, cols: usize, spread: f32) -> Vec<f32> {
        let mut v = vec![0f32; rows * cols];
        let mut s = 0x2545_F491_4F6C_DD1Du64;
        let mut next = || {
            s ^= s << 13;
            s ^= s >> 7;
            s ^= s << 17;
            (s >> 40) as f32 / 8388608.0 - 1.0
        };
        for r in 0..rows {
            let amp = 10f32.powi((r % 7) as i32 - 3); // rows span 10^-3..10^3
            for g in 0..cols / GROUP_SIZE {
                let tilt = (next().abs() * spread).exp2();
                for k in 0..GROUP_SIZE {
                    v[r * cols + g * GROUP_SIZE + k] = next() * amp * tilt;
                }
            }
        }
        v
    }

    fn rel_rms(a: &[f32], b: &[f32]) -> f32 {
        let (mut num, mut den) = (0f64, 0f64);
        for (x, y) in a.iter().zip(b) {
            num += ((x - y) as f64).powi(2);
            den += (*x as f64).powi(2);
        }
        (num / den).sqrt() as f32
    }

    /// Rows spanning 10^-3..10^3 with a per-group tilt, so the row ladder
    /// has to work for the scales to come back at all.
    fn q2tp_test_matrix(rows: usize, cols: usize) -> Vec<f32> {
        let mut v = vec![0f32; rows * cols];
        let mut s = 0x2545F491_4F6CDD1Du64;
        let mut next = || {
            s ^= s << 13;
            s ^= s >> 7;
            s ^= s << 17;
            (s >> 40) as f32 / 8388608.0 - 1.0
        };
        for r in 0..rows {
            let amp = 10f32.powi((r % 7) as i32 - 3);
            for g in 0..cols / GROUP_SIZE {
                let tilt = (next().abs() * 2.0).exp2();
                for k in 0..GROUP_SIZE {
                    v[r * cols + g * GROUP_SIZE + k] = next() * amp * tilt;
                }
            }
        }
        v
    }

    fn q2tp_rel_rms(a: &[f32], b: &[f32]) -> f32 {
        let (mut num, mut den) = (0f64, 0f64);
        for (x, y) in a.iter().zip(b) {
            num += ((x - y) as f64).powi(2);
            den += (*x as f64).powi(2);
        }
        (num / den).sqrt() as f32
    }

    /// The 2-bit plane must be exactly half of q4tp's, with the params and
    /// code planes untouched — a wrong section offset reads scales out of
    /// weight bytes and still produces fluent-looking numbers, so pin the
    /// length AND the reader's own view of it.
    /// E4M3 has no infinities and one NaN; the values below are the
    /// bit patterns that pin its exponent bias and its subnormal step.
    /// DeepSeek-V4 names arrive with no `model.` wrapper and spell the
    /// blocks `attn`/`ffn`. Pin the whole map: a silent miss here means a
    /// tensor lands under a name nothing reads, and the model loads with
    /// a hole instead of failing.
    /// The 2-bit profile is worth 50 GB on a 300B MoE, and getting it wrong
    /// produces a file that is merely large — no error, no warning. These are
    /// the names it has to recognize, in the spelling `canon_name` emits.
    /// The 2-bit levels are (c−1.5)·s, so the scale decides everything, and
    /// absmax/1.5 pins the outer level to the largest weight — which
    /// minimises the WORST error, not the mean one. With four levels that
    /// difference is large. This measures both arms on weights shaped like a
    /// real expert plane and requires the search to actually win.
    #[test]
    fn two_bit_scale_search_beats_the_absmax_rule() {
        let (rows, cols) = (16usize, 512usize);
        let mut w = vec![0.0f32; rows * cols];
        for (i, v) in w.iter_mut().enumerate() {
            let x = i as f32;
            *v = ((x * 0.017).sin() + (x * 0.041).sin() + (x * 0.093).sin()) / 3.0;
            if i % 97 == 0 {
                *v *= 6.0; // an outlier every so often, as in a real plane
            }
        }
        let rel = |enc: &[u8]| {
            let mut back = vec![0.0f32; rows * cols];
            cortiq_core::quant::dequant_q2tp(enc, rows, cols, &mut back);
            let num: f32 = w.iter().zip(&back).map(|(a, b)| (a - b) * (a - b)).sum();
            let den: f32 = w.iter().map(|a| a * a).sum();
            (num / den).sqrt()
        };
        // SAFETY-of-test: read per call, and these run in sequence.
        let _g = env_guard("1", "0");
        let plain = rel(&encode_q2tp(&w, rows, cols));
        unsafe { std::env::set_var("CMF_Q2TP_SEARCH", "1") };
        let searched = rel(&encode_q2tp(&w, rows, cols));
        println!("q2tp относительная ошибка: absmax {plain:.4} → подбор {searched:.4}");
        assert!(
            searched < plain,
            "подбор ступени не улучшил: {searched:.4} против {plain:.4}"
        );
    }

    /// The same trick in the 4-bit layout. Sixteen levels forgive an outlier
    /// far more than four do, so this measures whether it is worth having
    /// there at all rather than assuming it is.
    #[test]
    fn four_bit_scale_search_measured() {
        let (rows, cols) = (16usize, 512usize);
        let mut w = vec![0.0f32; rows * cols];
        for (i, v) in w.iter_mut().enumerate() {
            let x = i as f32;
            *v = ((x * 0.017).sin() + (x * 0.041).sin() + (x * 0.093).sin()) / 3.0;
            if i % 97 == 0 {
                *v *= 6.0;
            }
        }
        let rel = |enc: &[u8]| {
            let mut back = vec![0.0f32; rows * cols];
            cortiq_core::quant::dequant_q4tp(enc, rows, cols, &mut back);
            let num: f32 = w.iter().zip(&back).map(|(a, b)| (a - b) * (a - b)).sum();
            let den: f32 = w.iter().map(|a| a * a).sum();
            (num / den).sqrt()
        };
        let _g = env_guard("1", "0");
        let plain = rel(&encode_q4tp(&w, rows, cols));
        unsafe { std::env::set_var("CMF_Q2TP_SEARCH", "1") };
        let searched = rel(&encode_q4tp(&w, rows, cols));
        println!("q4tp относительная ошибка: absmax {plain:.4} → подбор {searched:.4}");
        assert!(searched <= plain, "подбор ухудшил 4 бита");
    }

    /// The row encoders were made parallel because a 300B MoE otherwise
    /// quantizes on one core. Rows are independent, so the bytes must be
    /// identical however many threads produce them — not merely close.
    #[test]
    fn parallel_row_encoding_is_byte_identical_to_serial() {
        // Wide enough to split across threads, with a dead row and a row of
        // wildly different magnitudes so the scale ladder actually varies.
        let (rows, cols) = (97usize, 256usize);
        let mut vals = vec![0.0f32; rows * cols];
        for r in 0..rows {
            for c in 0..cols {
                vals[r * cols + c] = match r {
                    7 => 0.0, // a dead row: rung 0 everywhere
                    _ => ((r * 31 + c * 7) as f32 * 0.017).sin() * (1 << (r % 9)) as f32,
                };
            }
        }
        let _g = env_guard("1", "1");
        for (name, enc) in [
            ("q2tp", encode_q2tp as fn(&[f32], usize, usize) -> Vec<u8>),
            ("q4tp", encode_q4tp as fn(&[f32], usize, usize) -> Vec<u8>),
            ("q8_2f", encode_q8_2f as fn(&[f32], usize, usize) -> Vec<u8>),
        ] {
            let serial = enc(&vals, rows, cols);
            unsafe { std::env::set_var("CMF_ENCODE_THREADS", "8") };
            let parallel = enc(&vals, rows, cols);
            unsafe { std::env::set_var("CMF_ENCODE_THREADS", "1") };
            assert_eq!(
                serial.len(),
                parallel.len(),
                "{name}: length differs between 1 and 8 threads"
            );
            let diff = serial.iter().zip(&parallel).position(|(a, b)| a != b);
            assert!(
                diff.is_none(),
                "{name}: byte {} differs between 1 and 8 threads",
                diff.unwrap()
            );
        }
    }

    #[test]
    fn the_two_bit_profile_covers_every_experts_gate_and_up() {
        let gu = q2tp_expert_gate_or_up;
        // routed experts — the bulk of the model
        assert!(gu("model.layers.7.mlp.experts.42.gate_proj.weight"));
        assert!(gu("model.layers.7.mlp.experts.42.up_proj.weight"));
        // the shared expert, which must match the routed layout
        assert!(gu("model.layers.7.mlp.shared_expert.gate_proj.weight"));
        assert!(gu("model.layers.7.mlp.shared_expert.up_proj.weight"));
        // down stays 4-bit, and nothing outside the experts is touched
        assert!(!gu("model.layers.7.mlp.experts.42.down_proj.weight"));
        assert!(!gu("model.layers.7.mlp.shared_expert.down_proj.weight"));
        assert!(!gu("model.layers.7.mlp.gate.weight"));
        assert!(!gu("model.layers.7.self_attn.wq_b.weight"));
        assert!(!gu("model.embed_tokens.weight"));
        // DeepSeek-V4 keeps the MTP layer's own ffn/w1,w3 spelling.
        assert!(gu("model.mtp.0.ffn.experts.42.w1.weight"));
        assert!(gu("model.mtp.2.ffn.experts.42.w3.weight"));
        assert!(gu("model.mtp.1.ffn.shared_experts.w1.weight"));
        assert!(gu("model.mtp.1.ffn.shared_experts.w3.weight"));
        assert!(!gu("model.mtp.0.ffn.experts.42.w2.weight"));
    }

    #[test]
    fn deepseek_v4_names_map_onto_the_loader_layout() {
        let m = |s: &str| canon_name(s).unwrap();
        assert_eq!(m("embed.weight"), "model.embed_tokens.weight");
        assert_eq!(m("head.weight"), "lm_head.weight");
        assert_eq!(m("norm.weight"), "model.norm.weight");
        assert_eq!(m("hc_head_scale"), "model.hc_head_scale");
        assert_eq!(
            m("layers.7.attn_norm.weight"),
            "model.layers.7.input_layernorm.weight"
        );
        assert_eq!(
            m("layers.7.ffn_norm.weight"),
            "model.layers.7.post_attention_layernorm.weight"
        );
        // router + the noaux_tc bias + the hash table
        assert_eq!(
            m("layers.7.ffn.gate.weight"),
            "model.layers.7.mlp.gate.weight"
        );
        assert_eq!(
            m("layers.7.ffn.gate.bias"),
            "model.layers.7.mlp.expert_bias"
        );
        assert_eq!(m("layers.7.ffn.gate.tid2eid"), "model.layers.7.mlp.tid2eid");
        // w1 = gate, w3 = up, w2 = down — for routed AND shared experts
        assert_eq!(
            m("layers.7.ffn.experts.42.w1.weight"),
            "model.layers.7.mlp.experts.42.gate_proj.weight"
        );
        assert_eq!(
            m("layers.7.ffn.experts.42.w3.weight"),
            "model.layers.7.mlp.experts.42.up_proj.weight"
        );
        assert_eq!(
            m("layers.7.ffn.experts.42.w2.scale"),
            "model.layers.7.mlp.experts.42.down_proj.scale"
        );
        assert_eq!(
            m("layers.7.ffn.shared_experts.w1.weight"),
            "model.layers.7.mlp.shared_expert.gate_proj.weight"
        );
        // attention keeps its own spelling under self_attn — the double
        // LoRA, the compressed KV, the sink, the compressor and the
        // indexer have no equivalent in a supported arch yet
        assert_eq!(
            m("layers.7.attn.wq_a.weight"),
            "model.layers.7.self_attn.wq_a.weight"
        );
        assert_eq!(
            m("layers.7.attn.indexer.weights_proj.weight"),
            "model.layers.7.self_attn.indexer.weights_proj.weight"
        );
        assert_eq!(
            m("layers.7.attn.compressor.wkv.weight"),
            "model.layers.7.self_attn.compressor.wkv.weight"
        );
        assert_eq!(m("layers.7.hc_attn_scale"), "model.layers.7.hc_attn_scale");
    }

    #[test]
    fn deepseek_v41_keeps_full_source_config_and_native_engram_names() {
        let config = serde_json::json!({
            "model_type": "deepseek_v41",
            "quantization_config": {
                "quant_method": "fp8",
                "weight_block_size": [32, 32],
                "scale_fmt": "ue8m0"
            },
            "text_config": {
                "model_type": "deepseek_v41_text",
                "hidden_size": 8,
                "intermediate_size": 16,
                "num_hidden_layers": 1,
                "num_attention_heads": 2,
                "num_key_value_heads": 1,
                "head_dim": 4,
                "qk_rope_head_dim": 2,
                "qk_nope_head_dim": 2,
                "vocab_size": 32,
                "rms_norm_eps": 1e-6,
                "n_routed_experts": 2,
                "moe_intermediate_size": 4,
                "num_experts_per_tok": 1,
                "n_shared_experts": 1
            },
            "vision_config": {
                "model_type": "deepseek_v41_vision",
                "hidden_size": 4,
                "num_hidden_layers": 2
            }
        });
        let arch = build_arch(&config).unwrap();
        assert_eq!(arch.arch_name, "deepseek_v41");
        assert_eq!(arch.deepseek_v41.as_ref(), Some(&config));
        assert_eq!(source_fp8_block(&config), 32);
        assert_eq!(
            canon_name("layers.1.engram.embed.weight").as_deref(),
            Some("model.layers.1.engram.embed.weight")
        );
        assert_eq!(
            canon_name("layers.1.engram.embed.scale").as_deref(),
            Some("model.layers.1.engram.embed.scale")
        );
        assert_eq!(
            canon_name("layers.1.ffn.gate.bias_vl").as_deref(),
            Some("model.layers.1.mlp.expert_bias_vl")
        );
    }

    #[test]
    fn dsv41_vision_and_aligner_matrix_weights_stay_f16_in_q4tp() {
        let config = serde_json::json!({
            "model_type": "deepseek_v41",
            "text_config": {
                "model_type": "deepseek_v41_text",
                "hidden_size": 8,
                "intermediate_size": 16,
                "num_hidden_layers": 1,
                "num_attention_heads": 2,
                "num_key_value_heads": 1,
                "head_dim": 4,
                "vocab_size": 32,
                "rms_norm_eps": 1e-6
            },
            "vision_config": {"hidden_size": 8}
        });
        let arch = build_arch(&config).unwrap();
        let values = vec![0.25f32; 8 * 8];
        let mut batches = vec![Vec::new()];
        for name in [
            "vision.patch_embed.proj.weight",
            "vision.blocks.0.attn.wqkv.weight",
            "vision.blocks.0.mlp.w1.weight",
            "aligner.w1.weight",
            "aligner.w2.weight",
        ] {
            emit_profiled_tensor(
                &mut batches,
                &[Quant::Q4TiledP],
                &[true],
                &arch,
                name,
                &[8, 8],
                &values,
                EmitMode::Auto,
            );
        }
        // The pre-existing F32 route remains authoritative when a control
        // tensor is encountered in the same profile.
        emit_profiled_tensor(
            &mut batches,
            &[Quant::Q4TiledP],
            &[true],
            &arch,
            "model.layers.0.mlp.expert_bias",
            &[8],
            &values[..8],
            EmitMode::Auto,
        );
        assert_eq!(
            batches[0][..5].iter().map(|t| t.dtype).collect::<Vec<_>>(),
            vec![
                TensorDtype::F16,
                TensorDtype::F16,
                TensorDtype::F16,
                TensorDtype::F16,
                TensorDtype::F16
            ]
        );
        assert_eq!(batches[0][5].dtype, TensorDtype::F32);
    }

    #[test]
    fn local_ready_consume_reports_unlink_failure() {
        let root = std::env::temp_dir().join(format!(
            "cortiq-local-ready-unlink-test-{}",
            std::process::id()
        ));
        let _ = fs::remove_dir_all(&root);
        fs::create_dir_all(&root).unwrap();
        let missing = root.join("already-removed.safetensors");
        let err = consume_local_ready_source(&missing)
            .expect_err("local-ready consume must fail when the source unlink fails");
        assert!(
            err.to_string()
                .contains("remove consumed local-ready source")
        );
        fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn fp8_e4m3_decodes_the_ocp_reference_points() {
        assert_eq!(fp8_e4m3_to_f32(0x00), 0.0);
        assert_eq!(fp8_e4m3_to_f32(0x38), 1.0); // exp 7 → 2^0
        assert_eq!(fp8_e4m3_to_f32(0xB8), -1.0);
        assert_eq!(fp8_e4m3_to_f32(0x3C), 1.5); // 1 + 4/8
        assert_eq!(fp8_e4m3_to_f32(0x40), 2.0);
        assert_eq!(fp8_e4m3_to_f32(0x01), (2f32).powi(-9)); // smallest subnormal
        assert_eq!(fp8_e4m3_to_f32(0x7E), 448.0); // largest finite
        assert!(fp8_e4m3_to_f32(0x7F).is_nan()); // E4M3FN NaN sentinel
        assert!(unpack_fp8_blocks(&[0x7F], &[127], 1, 1, 1).is_err());
    }

    /// The block plane must be indexed by TILE, not by element: a wrong
    /// stride still produces plausible numbers, which is how a silently
    /// wrong conversion happens.
    #[test]
    fn fp8_block_scales_apply_per_tile() {
        let (rows, cols, block) = (4usize, 4usize, 2usize);
        // every weight is 1.0; scales differ per 2x2 tile: 2^0, 2^1 / 2^2, 2^3
        let packed = vec![0x38u8; rows * cols];
        let scales = vec![127u8, 128, 129, 130];
        let out = unpack_fp8_blocks(&packed, &scales, rows, cols, block).unwrap();
        assert_eq!(out[0], 1.0); // tile (0,0)
        assert_eq!(out[2], 2.0); // tile (0,1)
        assert_eq!(out[2 * cols], 4.0); // tile (1,0)
        assert_eq!(out[2 * cols + 2], 8.0); // tile (1,1)
        // a NaN scale must fail rather than poison the tensor
        let bad = vec![255u8, 128, 129, 130];
        assert!(unpack_fp8_blocks(&packed, &bad, rows, cols, block).is_err());
    }

    #[test]
    fn fp8_f32_scale_inv_applies_per_block() {
        // 0x38 is exactly 1.0 in E4M3, so the output exposes the scale plane.
        let out = unpack_fp8_scale_inv(&[0x38; 4], &[1.0, 2.0, 3.0, 4.0], 2, 2, 1).unwrap();
        assert_eq!(out, vec![1.0, 2.0, 3.0, 4.0]);
        assert!(unpack_fp8_scale_inv(&[0x38; 4], &[1.0], 2, 2, 1).is_err());
        assert!(unpack_fp8_scale_inv(&[0x38; 4], &[f32::NAN; 4], 2, 2, 1).is_err());
        assert!(unpack_fp8_scale_inv(&[0x7F; 4], &[1.0; 4], 2, 2, 1).is_err());
    }

    /// Fine-grained FP8 decode is row-independent just like the output
    /// encoders. Keep a non-trivial tail row/column shape here so a future
    /// parallel rewrite cannot accidentally change block indexing or bytes
    /// while still passing a square toy case.
    #[test]
    fn fp8_scale_inv_decode_is_thread_count_invariant() {
        let (rows, cols, block) = (17usize, 130usize, 8usize);
        let sr = rows.div_ceil(block);
        let sc = cols.div_ceil(block);
        let packed: Vec<u8> = (0..rows * cols)
            .map(|i| ((i * 37 + 11) as u8) & 0x7e)
            .collect();
        let scales: Vec<f32> = (0..sr * sc)
            .map(|i| 0.125 + (i % 13) as f32 * 0.25)
            .collect();
        let _g = env_guard("1", "1");
        let serial = unpack_fp8_scale_inv(&packed, &scales, rows, cols, block).unwrap();
        unsafe { std::env::set_var("CMF_ENCODE_THREADS", "8") };
        let parallel = unpack_fp8_scale_inv(&packed, &scales, rows, cols, block).unwrap();
        assert_eq!(
            serial, parallel,
            "FP8 scale-inverse decode changed with threads"
        );
    }

    #[test]
    fn q2tp_payload_length_and_sections_match_the_reader() {
        use cortiq_core::quant::{expected_nbytes, q2tp_sections, validate_payload};
        let (rows, cols) = (7usize, 96usize);
        let v = q2tp_test_matrix(rows, cols);
        let blob = encode_q2tp(&v, rows, cols);
        let want = expected_nbytes(TensorDtype::Q2TiledP, &[rows, cols]).unwrap();
        assert_eq!(blob.len(), want, "payload length");
        let (params_off, codes_off, stride) = q2tp_sections(rows, cols);
        assert_eq!(params_off, rows * (cols / GROUP_SIZE) * 8, "weight plane");
        assert_eq!(codes_off, params_off + rows * 4, "params plane");
        assert_eq!(codes_off + rows * stride, blob.len(), "code plane");
        assert!(
            validate_payload(TensorDtype::Q2TiledP, &[rows, cols], &blob).is_ok(),
            "reader rejected its own encoder's output"
        );
    }

    /// 2 bits carry 4 levels, so the error is large by construction — what
    /// must hold is that it stays in the 2-bit BAND (a broken decode lands
    /// at ~100%) and that an all-zero row survives exactly.
    #[test]
    fn q2tp_roundtrip_stays_in_the_two_bit_error_band() {
        use cortiq_core::quant::dequant_q2tp;
        let (rows, cols) = (9usize, 128usize);
        let mut v = q2tp_test_matrix(rows, cols);
        for k in 0..cols {
            v[3 * cols + k] = 0.0;
        }
        let blob = encode_q2tp(&v, rows, cols);
        let mut back = vec![0f32; rows * cols];
        dequant_q2tp(&blob, rows, cols, &mut back);
        let e = q2tp_rel_rms(&back, &v);
        assert!(
            e < 0.55,
            "q2tp rel-RMS {e} — decode is wrong, not just coarse"
        );
        assert!(e > 0.05, "q2tp rel-RMS {e} — suspiciously exact for 2 bits");
        assert!(
            back[3 * cols..4 * cols].iter().all(|x| *x == 0.0),
            "an all-zero row must dequantize to exact zeros"
        );
    }

    /// Weights shaped like a real gate/up plane: heavy-tailed (a Gaussian
    /// scale mixture), a log-normal spread of row scales, per-channel
    /// amplitude drift, and rare outliers of `outlier`–1.7·`outlier` × RMS.
    fn q2tp_realistic_matrix(rows: usize, cols: usize, seed: u64) -> Vec<f32> {
        q2tp_synth_matrix(rows, cols, seed, 12.0)
    }

    fn q2tp_synth_matrix(rows: usize, cols: usize, seed: u64, outlier: f32) -> Vec<f32> {
        let mut s = seed | 1;
        let mut uni = || {
            s ^= s << 13;
            s ^= s >> 7;
            s ^= s << 17;
            ((s >> 11) as f64 + 0.5) / (1u64 << 53) as f64
        };
        let mut gauss = || {
            let (a, b) = (uni(), uni());
            ((-2.0 * a.ln()).sqrt() * (std::f64::consts::TAU * b).cos()) as f32
        };
        let chan: Vec<f32> = (0..cols).map(|_| (0.35 * gauss()).exp()).collect();
        let mut v = vec![0f32; rows * cols];
        for r in 0..rows {
            let row_scale = 0.02 * (0.5 * gauss()).exp();
            for c in 0..cols {
                let mix = if (r * 131 + c * 31) % 7 == 0 { 2.2 } else { 1.0 };
                let mut x = gauss() * mix * chan[c] * row_scale;
                if (r * 977 + c * 613) % 499 == 0 {
                    x *= outlier * (1.0 + (c % 3) as f32 / 3.0);
                }
                v[r * cols + c] = x;
            }
        }
        v
    }

    fn q2tp_decode(blob: &[u8], rows: usize, cols: usize) -> Vec<f32> {
        let mut back = vec![0f32; rows * cols];
        cortiq_core::quant::dequant_q2tp(blob, rows, cols, &mut back);
        back
    }

    fn sq_err(a: &[f32], b: &[f32]) -> f64 {
        a.iter().zip(b).map(|(x, y)| ((x - y) as f64).powi(2)).sum()
    }

    /// The 2-bit rounding helper must equal `round_ties_even().clamp(0, 3)`
    /// — ties, negatives, the clamp edges and far outliers (a −0 result may
    /// come back as +0: same code, same product).
    #[test]
    fn q2_level_is_round_ties_even_clamped() {
        let mut xs: Vec<f32> = (-4000..4000).map(|k| k as f32 / 1000.0).collect();
        xs.extend([-0.5, 0.5, 1.5, 2.5, 3.5, 4.5, -1e30, 1e30, f32::MIN_POSITIVE, -0.0, 0.0]);
        xs.extend((0..64).map(|k| 0.5 + k as f32 * 0.5 + f32::EPSILON * k as f32));
        xs.extend((0..64).map(|k| k as f32 * 0.5 - 0.5 - f32::EPSILON));
        for x in xs {
            let want = x.round_ties_even().clamp(0.0, 3.0);
            let got = q2_level(x);
            assert!(got == want && got.to_bits() & 0x7FFF_FFFF == want.to_bits() & 0x7FFF_FFFF, "x = {x}");
            assert_eq!(got as u8, want as u8, "x = {x}");
        }
    }

    /// v2 must cut the weight error against v1 on weights shaped like real
    /// gate/up planes — after its own debias, which by construction gives
    /// back ~1/b of the error it removed — and keep the payload layout.
    #[test]
    fn q2tp_v2_weight_mse_beats_v1_on_realistic_tensors() {
        for (rows, cols, seed) in [(64usize, 2048usize, 7u64), (48, 1024, 0x9E37), (32, 4096, 99)] {
            let w = q2tp_realistic_matrix(rows, cols, seed);
            let v1 = encode_q2tp_v1(&w, rows, cols, true);
            let v2 = encode_q2tp_v2(&w, rows, cols);
            assert_eq!(v1.len(), v2.len(), "same bytes per tensor");
            let (e1, e2) = (sq_err(&w, &q2tp_decode(&v1, rows, cols)), sq_err(&w, &q2tp_decode(&v2, rows, cols)));
            println!("q2tp {rows}x{cols}: weight SSE v1 {e1:.4e} → v2 {e2:.4e} ({:+.1} %)", 100.0 * (e2 / e1 - 1.0));
            assert!(e2 < 0.9 * e1, "v2 {e2:.4e} must beat v1 {e1:.4e} by >10 %");
        }
        // The q2tp test matrix (rows spanning 10^-3..10^3, tilted groups) too.
        let (rows, cols) = (21usize, 512usize);
        let w = q2tp_test_matrix(rows, cols);
        let e1 = sq_err(&w, &q2tp_decode(&encode_q2tp_v1(&w, rows, cols, true), rows, cols));
        let e2 = sq_err(&w, &q2tp_decode(&encode_q2tp_v2(&w, rows, cols), rows, cols));
        assert!(e2 < e1, "v2 {e2:.4e} vs v1 {e1:.4e} on the tilted matrix");
    }

    /// The placed ladder's codes are MSE-optimal, so the rows come back
    /// shrunk; the debias step must bring each row's slope back to ~1 (to
    /// within the f16 `lo` granularity; the gain is clipped to 0.8..1.25)
    /// without touching codes or rungs.
    #[test]
    fn q2tp_v2_debias_restores_unit_slope_and_keeps_codes() {
        let (rows, cols) = (24usize, 1024usize);
        let w = q2tp_synth_matrix(rows, cols, 1234, 3.0);
        let v2 = encode_q2tp_v2(&w, rows, cols);
        // The same placement + RTN without the debias step.
        let gpr = cols / GROUP_SIZE;
        let ones = vec![1f32; cols];
        let (po, co, stride) = cortiq_core::quant::q2tp_sections(rows, cols);
        let mut raw = vec![0u8; v2.len()];
        for r in 0..rows {
            let row = &w[r * cols..(r + 1) * cols];
            let mut rungs = vec![0u8; gpr];
            let (chunks, rest) = raw.split_at_mut(po);
            let (params, codes) = rest.split_at_mut(rows * 4);
            q2tp_place_row(row, &ones, &mut params[r * 4..r * 4 + 4], &mut rungs);
            q2tp_pack_rtn(
                row,
                &params[r * 4..r * 4 + 4],
                &rungs,
                &mut chunks[r * gpr * Q2TP_CHUNK..(r + 1) * gpr * Q2TP_CHUNK],
                &mut codes[r * stride..(r + 1) * stride],
            );
        }
        assert_eq!(&v2[..po], &raw[..po], "2-bit plane unchanged by the debias");
        assert_eq!(&v2[co..], &raw[co..], "rung codes unchanged by the debias");
        for r in 0..rows {
            assert_eq!(v2[po + r * 4 + 2..po + r * 4 + 4], raw[po + r * 4 + 2..po + r * 4 + 4], "step unchanged");
        }
        let slope = |blob: &[u8], r: usize| -> f64 {
            let d = q2tp_decode(blob, rows, cols);
            let (mut n, mut dd) = (0f64, 0f64);
            for c in 0..cols {
                let x = w[r * cols + c] as f64;
                n += x * d[r * cols + c] as f64;
                dd += x * x;
            }
            n / dd
        };
        let (mut shrunk, mut unit) = (0usize, 0usize);
        for r in 0..rows {
            let (b0, b2) = (slope(&raw, r), slope(&v2, r));
            shrunk += (b0 < 0.99) as usize;
            let want = b0 / b0.clamp(0.8, 1.25);
            assert!((b2 - want).abs() < 0.01, "row {r}: slope {b0:.4} → {b2:.4} (want {want:.4})");
            unit += ((b2 - 1.0).abs() < 0.01) as usize;
        }
        assert!(shrunk > rows / 2, "MSE-optimal codes are expected to shrink most rows");
        assert!(unit > rows / 2, "most rows must come back to a unit slope ({unit}/{rows})");
    }

    /// `CMF_Q2TP_ENCODER=v1` is the escape hatch: byte-identical to the old
    /// encoder; the default is v2; `CMF_Q2TP_SEARCH=0` stays the absmax arm.
    #[test]
    fn q2tp_encoder_switch_selects_v1_or_v2() {
        let (rows, cols) = (11usize, 256usize);
        let w = q2tp_realistic_matrix(rows, cols, 5);
        let _g = env_guard("1", "1");
        unsafe { std::env::remove_var("CMF_Q2TP_ENCODER") };
        assert_eq!(encode_q2tp(&w, rows, cols), encode_q2tp_v2(&w, rows, cols), "default = v2");
        unsafe { std::env::set_var("CMF_Q2TP_ENCODER", "v1") };
        assert_eq!(encode_q2tp(&w, rows, cols), encode_q2tp_v1(&w, rows, cols, true), "v1 fallback");
        unsafe {
            std::env::remove_var("CMF_Q2TP_ENCODER");
            std::env::set_var("CMF_Q2TP_SEARCH", "0");
        }
        assert_eq!(encode_q2tp(&w, rows, cols), encode_q2tp_v1(&w, rows, cols, false), "search off");
        unsafe { std::env::set_var("CMF_Q2TP_SEARCH", "1") };
    }

    /// v2 with many threads writes the same bytes as with one, and every
    /// payload it writes passes the reader's own validation.
    #[test]
    fn q2tp_v2_parallel_bytes_and_reader_validation() {
        use cortiq_core::quant::validate_payload;
        let (rows, cols) = (37usize, 512usize);
        let mut w = q2tp_realistic_matrix(rows, cols, 77);
        w[5 * cols..6 * cols].fill(0.0); // a dead row
        w[9 * cols + 32..9 * cols + 64].fill(0.0); // a dead group
        let _g = env_guard("1", "1");
        let serial = encode_q2tp_v2(&w, rows, cols);
        unsafe { std::env::set_var("CMF_ENCODE_THREADS", "8") };
        let parallel = encode_q2tp_v2(&w, rows, cols);
        unsafe { std::env::set_var("CMF_ENCODE_THREADS", "1") };
        assert_eq!(serial, parallel, "v2 bytes depend on the thread count");
        assert!(validate_payload(TensorDtype::Q2TiledP, &[rows, cols], &serial).is_ok());
        let back = q2tp_decode(&serial, rows, cols);
        assert!(back[5 * cols..6 * cols].iter().all(|x| *x == 0.0), "dead row");
        assert!(back[9 * cols + 32..9 * cols + 64].iter().all(|x| *x == 0.0), "dead group");
    }

    #[test]
    fn q4tp_payload_length_matches_expected_nbytes() {
        for &(rows, cols) in &[(1usize, 32usize), (7, 64), (16, 2048), (2048, 512)] {
            let v = synth_rows(rows, cols, 1.27);
            let bytes = encode_q4tp(&v, rows, cols);
            assert_eq!(
                Some(bytes.len()),
                expected_nbytes(TensorDtype::Q4TiledP, &[rows, cols]),
                "shape {rows}x{cols}"
            );
        }
    }

    #[test]
    fn q4tp_codes_survive_bit_packing_at_every_offset() {
        // 5 bits per code means the byte boundary lands differently for each
        // tile mod 8; walk a full period plus change.
        let gpr = 37;
        let mut buf = vec![0u8; q4tp_code_stride(gpr)];
        let want: Vec<usize> = (0..gpr).map(|g| (g * 7 + 3) % 32).collect();
        for (g, &c) in want.iter().enumerate() {
            q4tp_put_code(&mut buf, g, c);
        }
        let got: Vec<usize> = (0..gpr)
            .map(|g| cortiq_core::quant::q4tp_code(&buf, g))
            .collect();
        assert_eq!(got, want);
    }

    #[test]
    fn q4tp_costs_almost_nothing_against_q4t_from_the_same_source() {
        // The honest comparison is BOTH codecs against the same fp32 weights.
        // Comparing q4tp against q4t's output instead measures the distance
        // between two representations, which reads far worse than the truth:
        // when the scale shifts a little the nibbles simply re-round, so the
        // total error stays pinned to the 4-bit grid either way.
        let (rows, cols) = (256usize, 2048usize);
        for &(spread, budget) in &[(0.0f32, 1.02f32), (1.27, 1.02), (1.89, 1.03), (3.70, 1.06)] {
            let v = synth_rows(rows, cols, spread);
            let mut a = vec![0f32; rows * cols];
            dequant_q4_tiled(&encode_q4_tiled(&v, rows, cols), &mut a);
            let mut b = vec![0f32; rows * cols];
            dequant_q4tp(&encode_q4tp(&v, rows, cols), rows, cols, &mut b);

            let (e_q4t, e_q4tp) = (rel_rms(&v, &a), rel_rms(&v, &b));
            assert!(
                e_q4tp <= e_q4t * budget,
                "spread {spread}: q4tp {e_q4tp:.4} over budget {budget}× of q4t {e_q4t:.4}"
            );
        }
    }

    #[test]
    fn q4tp_keeps_an_all_zero_tile_exactly_zero() {
        // A dead tile must not drag the row's ladder down (that would coarsen
        // every live tile) and must still dequantize to exact zeros.
        let (rows, cols) = (4usize, 128usize);
        let mut v = synth_rows(rows, cols, 1.27);
        for c in 0..GROUP_SIZE {
            v[cols + c] = 0.0;
        }
        let mut out = vec![0f32; rows * cols];
        dequant_q4tp(&encode_q4tp(&v, rows, cols), rows, cols, &mut out);
        assert!(out[cols..cols + GROUP_SIZE].iter().all(|&x| x == 0.0));

        let live = &v[cols + GROUP_SIZE..2 * cols];
        let got = &out[cols + GROUP_SIZE..2 * cols];
        assert!(
            rel_rms(live, got) < 0.15,
            "dead tile stretched the row's ladder"
        );
    }

    #[test]
    fn granite_42_maps_scaling_and_protects_vocabulary_edges() {
        let config = serde_json::json!({
            "architectures": ["GraniteForCausalLM"],
            "model_type": "granite",
            "hidden_size": 4096,
            "intermediate_size": 32768,
            "num_hidden_layers": 64,
            "num_attention_heads": 32,
            "num_key_value_heads": 8,
            "vocab_size": 100352,
            "max_position_embeddings": 131072,
            "rms_norm_eps": 0.00001,
            "rope_theta": 50000000.0,
            "attention_multiplier": 0.0078125,
            "embedding_multiplier": 1.0,
            "residual_multiplier": 1.0,
            "logits_scaling": 1.0,
            "hidden_act": "silu",
            "tie_word_embeddings": false
        });
        let arch = build_arch(&config).unwrap();
        assert_eq!(arch.arch_name, "granite");
        assert_eq!((arch.hidden_size, arch.head_dim), (4096, 128));
        assert_eq!(arch.rope_theta, 50_000_000.0);
        assert_eq!(arch.norm_style, NormStyle::Qwen);
        assert_eq!(arch.embed_multiplier, 1.0);
        assert_eq!(arch.logit_multiplier, None);
        assert_eq!(arch.query_pre_attn_scalar, Some(16_384.0));
        assert_eq!(
            quant_for_tensor(&arch, "model.embed_tokens.weight", Quant::Q4TiledP),
            Quant::Q8_2f
        );
        assert_eq!(
            quant_for_tensor(&arch, "lm_head.weight", Quant::Q4TiledP),
            Quant::Q8_2f
        );
        assert_eq!(
            quant_for_tensor(
                &arch,
                "model.layers.0.self_attn.q_proj.weight",
                Quant::Q4TiledP,
            ),
            Quant::Q4TiledP
        );
    }

    #[test]
    fn granite_refuses_an_unimplemented_residual_multiplier() {
        let config = serde_json::json!({
            "model_type": "granite",
            "hidden_size": 64,
            "intermediate_size": 128,
            "num_hidden_layers": 1,
            "num_attention_heads": 4,
            "num_key_value_heads": 2,
            "vocab_size": 32,
            "residual_multiplier": 0.22
        });
        let err = build_arch(&config).unwrap_err().to_string();
        assert!(err.contains("residual_multiplier=0.22"), "{err}");
    }

    #[test]
    fn laguna_config_maps_to_exact_cmf_contract() {
        let config = serde_json::json!({
            "model_type": "laguna",
            "hidden_size": 32,
            "intermediate_size": 64,
            "num_hidden_layers": 4,
            "num_attention_heads": 4,
            "num_attention_heads_per_layer": [4, 6, 6, 6],
            "num_key_value_heads": 2,
            "head_dim": 8,
            "vocab_size": 100,
            "max_position_embeddings": 1048576,
            "num_experts": 8,
            "num_experts_per_tok": 2,
            "moe_intermediate_size": 16,
            "shared_expert_intermediate_size": 16,
            "norm_topk_prob": true,
            "moe_routed_scaling_factor": 2.5,
            "sliding_window": 512,
            "layer_types": ["full_attention", "sliding_attention", "sliding_attention", "sliding_attention"],
            "rope_parameters": {
                "full_attention": {
                    "rope_type": "yarn", "rope_theta": 500000.0,
                    "factor": 128.0, "original_max_position_embeddings": 8192,
                    "beta_fast": 32.0, "beta_slow": 1.0,
                    "attention_factor": 1.485203, "partial_rotary_factor": 0.5
                },
                "sliding_attention": {
                    "rope_type": "default", "rope_theta": 10000.0,
                    "partial_rotary_factor": 1.0
                }
            }
        });
        let arch = build_arch(&config).unwrap();
        assert_eq!(arch.arch_name, "laguna");
        assert_eq!(arch.attention_heads_per_layer, Some(vec![4, 6, 6, 6]));
        assert_eq!(
            arch.layer_types,
            vec![
                LayerType::FullAttention,
                LayerType::SlidingAttention,
                LayerType::SlidingAttention,
                LayerType::SlidingAttention,
            ]
        );
        assert_eq!(arch.sliding_window, Some(512));
        assert_eq!(arch.rope_theta, 500000.0);
        assert_eq!(arch.rope_local_base_freq, Some(10000.0));
        assert_eq!(arch.partial_rotary_factor, 0.5);
        assert_eq!(arch.local_partial_rotary_factor, Some(1.0));
        let yarn = arch.yarn.unwrap();
        assert_eq!(yarn.factor, 128.0);
        let moe = arch.moe.unwrap();
        assert!(moe.router_sigmoid);
        assert_eq!(moe.routed_scaling_factor, Some(2.5));
        assert_eq!(
            canon_name("model.layers.1.mlp.experts.e_score_correction_bias").as_deref(),
            Some("model.layers.1.mlp.expert_bias")
        );
    }

    #[test]
    fn mellum_config_keeps_dual_rope_swa_and_softmax_moe_contract() {
        // JetBrains/Mellum2.1-12B-A2.5B-Thinking has three local layers
        // followed by one full-context layer.  Its full layers use YaRN,
        // while local layers deliberately stay on ordinary 500k-base RoPE.
        // Keep the fixture small, but retain every architecture switch from
        // the release config that affects a CMF forward pass.
        let config = serde_json::json!({
            "model_type": "mellum",
            "hidden_size": 32,
            "intermediate_size": 64,
            "num_hidden_layers": 4,
            "num_attention_heads": 4,
            "num_key_value_heads": 1,
            "head_dim": 8,
            "vocab_size": 100,
            "max_position_embeddings": 131072,
            "rms_norm_eps": 0.000001,
            "hidden_act": "silu",
            "tie_word_embeddings": false,
            "sliding_window": 1024,
            "layer_types": [
                "sliding_attention", "sliding_attention",
                "sliding_attention", "full_attention"
            ],
            "num_experts": 64,
            "num_experts_per_tok": 8,
            "moe_intermediate_size": 16,
            "norm_topk_prob": true,
            "rope_parameters": {
                "full_attention": {
                    "rope_type": "yarn",
                    "rope_theta": 500000.0,
                    "factor": 16.0,
                    "original_max_position_embeddings": 8192,
                    "beta_fast": 32.0,
                    "beta_slow": 1.0,
                    // This is applied by the full-attention YaRN profile;
                    // the independent sliding default-RoPE profile stays 1.
                    "attention_factor": 1.2772588722239782
                },
                "sliding_attention": {
                    "rope_type": "default",
                    "rope_theta": 500000.0
                }
            }
        });

        let arch = build_arch(&config).unwrap();
        assert_eq!(arch.arch_name, "mellum");
        assert_eq!(arch.max_position_embeddings, 131072);
        assert_eq!(arch.norm_style, NormStyle::Qwen);
        assert_eq!(
            arch.layer_types,
            vec![
                LayerType::SlidingAttention,
                LayerType::SlidingAttention,
                LayerType::SlidingAttention,
                LayerType::FullAttention,
            ]
        );
        assert_eq!(arch.sliding_window, Some(1024));
        assert_eq!(arch.sliding_window_pattern, None);
        assert_eq!(arch.rope_theta, 500000.0);
        assert_eq!(arch.rope_local_base_freq, Some(500000.0));
        assert_eq!(arch.partial_rotary_factor, 1.0);
        assert_eq!(arch.local_partial_rotary_factor, None);
        let yarn = arch.yarn.as_ref().unwrap();
        assert_eq!(yarn.factor, 16.0);
        assert_eq!(yarn.original_max_position_embeddings, 8192);
        assert_eq!(yarn.beta_fast, 32.0);
        assert_eq!(yarn.beta_slow, 1.0);
        assert_eq!(yarn.attention_factor, 1.2772588722239782);
        let moe = arch.moe.as_ref().unwrap();
        assert_eq!(moe.num_experts, 64);
        assert_eq!(moe.top_k, 8);
        assert_eq!(moe.moe_intermediate_size, 16);
        assert!(moe.norm_topk_prob);
        assert!(!moe.router_sigmoid);
        assert_eq!(moe.shared_expert_intermediate_size, None);

        // Mellum's 64-expert bank is the memory wall: both `auto` and an
        // explicit q4tp request use q4tp for it, but retain every token's
        // attention/vocabulary skeleton at q8_2f.  The router is protected
        // independently by the f16 force rule; vector norms take the f16
        // emitter path.
        assert_eq!(default_quant_for_arch(&arch), Quant::Q4TiledP);
        for requested in [default_quant_for_arch(&arch), Quant::Q4TiledP] {
            assert_eq!(
                profile_quant(&arch, requested, "model.layers.0.mlp.experts.3.gate_proj.weight"),
                Quant::Q4TiledP
            );
            assert_eq!(
                profile_quant(&arch, requested, "model.layers.0.mlp.experts.3.up_proj.weight"),
                Quant::Q4TiledP
            );
            assert_eq!(
                profile_quant(&arch, requested, "model.layers.0.mlp.experts.3.down_proj.weight"),
                Quant::Q4TiledP
            );
            assert_eq!(
                profile_quant(&arch, requested, "model.layers.0.self_attn.q_proj.weight"),
                Quant::Q8_2f
            );
            assert_eq!(
                profile_quant(&arch, requested, "model.embed_tokens.weight"),
                Quant::Q8_2f
            );
            assert_eq!(profile_quant(&arch, requested, "lm_head.weight"), Quant::Q8_2f);
        }
        assert!(force_f16("model.layers.0.mlp.gate.weight"));
        assert_eq!(
            profile_quant(
                &arch,
                Quant::F16,
                "model.layers.0.mlp.experts.3.gate_proj.weight"
            ),
            Quant::F16,
            "an explicit f16 conversion remains an exact-parity path"
        );
    }

    #[test]
    fn exec_order_lays_out_by_layer_then_block() {
        // Alphabetical order (the safetensors default) would put layer 10 before
        // layer 2 and the router after the experts — the exec-order key fixes both.
        let mut names: Vec<&str> = vec![
            "lm_head.weight",
            "model.layers.10.mlp.experts.1.up_proj.weight",
            "model.embed_tokens.weight",
            "model.layers.2.self_attn.q_proj.weight",
            "model.norm.weight",
            "model.layers.2.mlp.gate.weight", // MoE router
            "model.layers.2.mlp.experts.0.down_proj.weight",
            "model.layers.2.input_layernorm.weight",
            "model.layers.2.mlp.experts.0.gate_proj.weight",
            "model.layers.10.self_attn.o_proj.weight",
        ];
        names.sort_by(|a, b| {
            exec_order_key(a)
                .cmp(&exec_order_key(b))
                .then_with(|| a.cmp(b))
        });
        assert_eq!(
            names,
            vec![
                "model.embed_tokens.weight",
                "model.layers.2.input_layernorm.weight",
                "model.layers.2.self_attn.q_proj.weight",
                "model.layers.2.mlp.gate.weight",
                "model.layers.2.mlp.experts.0.gate_proj.weight",
                "model.layers.2.mlp.experts.0.down_proj.weight",
                "model.layers.10.self_attn.o_proj.weight",
                "model.layers.10.mlp.experts.1.up_proj.weight",
                "model.norm.weight",
                "lm_head.weight",
            ]
        );
    }

    #[test]
    fn vbit_roundtrip_within_quant_error() {
        // rows with distinct amplitudes → distinct bit-widths; 2 groups per row.
        let (rows, cols) = (5usize, 64usize);
        let mut vals = vec![0f32; rows * cols];
        for o in 0..rows {
            for i in 0..cols {
                vals[o * cols + i] = (o as f32 + 1.0) * 0.13 * (i as f32 * 0.27).sin();
            }
        }
        let enc = encode_vbit(&vals, rows, cols);
        // header sizes match the decoder's expectation.
        let bits = &enc[..rows];
        assert!(bits.iter().all(|&b| (3..=8).contains(&b)));
        let mut dec = vec![0f32; rows * cols];
        dequant_vbit(&enc, rows, cols, &mut dec).unwrap();
        for o in 0..rows {
            let amp = vals[o * cols..(o + 1) * cols]
                .iter()
                .fold(0f32, |m, v| m.max(v.abs()))
                .max(1e-6);
            for i in 0..cols {
                let e = (dec[o * cols + i] - vals[o * cols + i]).abs();
                assert!(
                    e <= amp * 0.2,
                    "row {o} col {i}: err {e} vs amp {amp} (bits {})",
                    bits[o]
                );
            }
        }
    }

    #[test]
    fn vbit_ro_roundtrip_and_validation() {
        use cortiq_core::TensorDtype;
        use cortiq_core::quant::{dequant_vbit_ro, validate_payload};
        let (rows, cols) = (5usize, 64usize);
        let mut vals = vec![0f32; rows * cols];
        for o in 0..rows {
            for i in 0..cols {
                vals[o * cols + i] = (o as f32 + 1.0) * 0.13 * (i as f32 * 0.27).sin();
            }
        }
        let enc = encode_vbit_ro(&vals, rows, cols);
        validate_payload(TensorDtype::VbitRo, &[rows, cols], &enc).unwrap();
        let mut dec = vec![0f32; rows * cols];
        dequant_vbit_ro(&enc, rows, cols, &mut dec).unwrap();
        // Must be BYTE-identical in reconstruction to the legacy layout.
        let legacy = encode_vbit(&vals, rows, cols);
        let mut dec_legacy = vec![0f32; rows * cols];
        dequant_vbit(&legacy, rows, cols, &mut dec_legacy).unwrap();
        assert_eq!(
            dec, dec_legacy,
            "vbit_ro must reconstruct exactly like vbit"
        );
    }

    #[test]
    fn tiktoken_roundtrip_synthetic() {
        use base64::Engine as _;
        let b64 = base64::engine::general_purpose::STANDARD;
        // Mini rank table: single bytes first, then pairs — exactly the
        // shape tiktoken ships (ranks are the merge order).
        let toks: Vec<&[u8]> = vec![
            b"h", b"e", b"l", b"o", b" ", b"w", b"r", b"d", b"he", b"ll", b"lo", b"hell", b"hello",
            b" w", b"or", b" wor", b" world",
        ];
        let model: String = toks
            .iter()
            .enumerate()
            .map(|(r, t)| format!("{} {}\n", b64.encode(t), r))
            .collect();
        let cfg = serde_json::json!({
            "added_tokens_decoder": {
                "100": {"content": "[BOS]"},
                "101": {"content": "[EOS]"}
            }
        });
        let json = tiktoken_to_tokenizer_json(&model, &cfg).unwrap();
        let tok = cortiq_engine::tokenizer::Tokenizer::from_json(&json).unwrap();
        let ids = tok.encode("hello world");
        // "hello" merges to one token (rank 12). " world"(16) is BPE-
        // UNREACHABLE with this table (merge path stops at " wor"+l+d) —
        // real tiktoken tokenizes it identically, which is exactly the
        // semantics the recovered merge list must reproduce.
        let strip: Vec<u32> = ids.iter().copied().filter(|&i| i != 100).collect(); // drop a possible auto-BOS
        assert_eq!(strip, vec![12, 15, 2, 7], "ids: {ids:?}");
        assert_eq!(tok.decode(&strip), "hello world");
    }

    #[test]
    fn qwen4_exp_arch_carries_portable_qsa_ple_geometry() {
        let cfg = serde_json::json!({
            "model_type": "qwen4_exp",
            "text_config": {
                "model_type": "qwen4_exp_text",
                "hidden_size": 2560,
                "moe_intermediate_size": 640,
                "num_hidden_layers": 48,
                "num_attention_heads": 24,
                "num_key_value_heads": 2,
                "head_dim": 256,
                "vocab_size": 248320,
                "max_position_embeddings": 262144,
                "partial_rotary_factor": 0.25,
                "rope_theta": 10000000,
                "num_experts": 512,
                "num_experts_per_tok": 10,
                "shared_expert_intermediate_size": 640,
                "linear_num_value_heads": 48,
                "linear_num_key_heads": 16,
                "linear_key_head_dim": 128,
                "linear_value_head_dim": 128,
                "linear_conv_kernel_dim": 4,
                "layer_types": ["linear_attention", "full_attention"],
                "hc_count": 4,
                "hc_lowrank": 320,
                "indexer_n_heads": 4,
                "indexer_kv_heads": 1,
                "indexer_head_dim": 128,
                "indexer_budget": 2048,
                "indexer_compress_ratio": 4,
                "ple_layer_ids": [2],
                "ple_embed_dim": 2560,
                "ple_conv_kernel_size": 4,
                "ngram_size": 3,
                "heads_per_ngram": 8,
                "ngram_vocab_size_base": 20000000,
                "make_ngram_vocab_size_divisible_by": 128,
                "split_ngram_parts": 128,
                "mtp_num_hidden_layers": 1
            }
        });
        let mut cfg = cfg;
        cfg["text_config"]["num_hidden_layers"] = 2.into();
        let mut arch = build_arch(&cfg).unwrap();
        let q = arch.qwen4_exp.as_ref().unwrap();
        assert_eq!(arch.norm_style, NormStyle::Gemma);
        assert_eq!(
            arch.layer_types,
            [LayerType::LinearAttention, LayerType::FullAttention]
        );
        assert_eq!((q.hc_count, q.hc_lowrank), (4, 320));
        assert_eq!((q.indexer_budget, q.indexer_compress_ratio), (2048, 4));
        assert_eq!(q.ple_layer_ids, [2]);
        assert_eq!(q.split_ngram_parts, 128);
        assert!(arch.moe.as_ref().unwrap().norm_topk_prob);
        assert!(arch.mtp.is_none(), "qwen4 MTP is a different stack");
        assert_eq!(
            quant_for_tensor(
                &arch,
                "model.layers.0.linear_attn.in_proj_qkv.weight",
                Quant::Q4TiledP,
            ),
            Quant::Q8_2f
        );
        assert_eq!(
            quant_for_tensor(
                &arch,
                "model.layers.0.mlp.experts.0.gate_proj.weight",
                Quant::Q4TiledP,
            ),
            Quant::Q4TiledP
        );
        assert_eq!(
            quant_for_tensor(
                &arch,
                "model.layers.1.ple.ple_embedding.ngram_embedding.shard_0.weight",
                Quant::Q4TiledP,
            ),
            Quant::Q4TiledP
        );
        assert!(force_f16(
            "model.hyper_connection_mixer.input_mix_weight_down.weight"
        ));
        arch.num_layers = 48;
        assert_eq!(
            directory_record_estimate(1, &arch),
            73_729,
            "the fixed streamed head must include packed MoE expansion"
        );
    }

    #[test]
    fn tiktoken_real_model_roundtrip() {
        // Gated: point CMF_TIKTOKEN at a real tiktoken.model (+ optional
        // CMF_TIKTOKEN_CFG tokenizer_config.json) to validate against the
        // shipped table. Skipped otherwise.
        let Ok(path) = std::env::var("CMF_TIKTOKEN") else {
            return;
        };
        let model = std::fs::read_to_string(&path).unwrap();
        let cfg: serde_json::Value = std::env::var("CMF_TIKTOKEN_CFG")
            .ok()
            .and_then(|p| std::fs::read_to_string(p).ok())
            .and_then(|s| serde_json::from_str(&s).ok())
            .unwrap_or(serde_json::Value::Null);
        let t0 = std::time::Instant::now();
        let json = tiktoken_to_tokenizer_json(&model, &cfg).unwrap();
        eprintln!(
            "converted in {:?}, json {} MB",
            t0.elapsed(),
            json.len() / 1_000_000
        );
        let tok = cortiq_engine::tokenizer::Tokenizer::from_json(&json).unwrap();
        for text in [
            "Hello, world!",
            "Kimi Linear is a hybrid attention architecture.",
            "Пример текста на русском языке 123",
            "你好，世界！混合注意力架构。",
            "def main():\n    print(\"ok\")  # comment\n",
            "   spaces    and\ttabs\n\n",
        ] {
            let ids = tok.encode(text);
            let back = tok.decode(
                &ids.iter()
                    .copied()
                    .filter(|&i| Some(i) != tok.bos_token_id)
                    .collect::<Vec<_>>(),
            );
            assert_eq!(back, text, "roundtrip failed for {text:?} → {ids:?}");
            eprintln!("{:3} toks | {text:?}", ids.len());
        }
    }

    #[test]
    fn mxfp4_unpack_roundtrip() {
        // Values ON the E2M1 grid times power-of-two scales roundtrip
        // exactly; low nibble = even element (compressed-tensors pack).
        let grid = [0.0f32, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0];
        let rows = 2usize;
        let cols = 64usize; // two groups of 32 per row
        let scales_exp = [[127u8, 130u8], [124u8, 127u8]]; // ×1, ×8, ×0.125, ×1
        let mut want = vec![0.0f32; rows * cols];
        let mut packed = vec![0u8; rows * cols / 2];
        let mut scales = vec![0u8; rows * 2];
        for r in 0..rows {
            for g in 0..2 {
                let k = scales_exp[r][g];
                scales[r * 2 + g] = k;
                let sc = (k as f32 - 127.0).exp2();
                for i in 0..32 {
                    let nib = ((r * 31 + g * 7 + i * 3) % 16) as u8;
                    let mag = grid[(nib & 7) as usize];
                    let v = if nib & 8 != 0 { -mag } else { mag };
                    want[r * cols + g * 32 + i] = v * sc;
                    let byte = &mut packed[r * cols / 2 + g * 16 + i / 2];
                    if i % 2 == 0 {
                        *byte |= nib;
                    } else {
                        *byte |= nib << 4;
                    }
                }
            }
        }
        let got = unpack_mxfp4(&packed, &scales, rows, cols / 2).unwrap();
        assert_eq!(got, want);
        // NaN scale (E8M0 255) must refuse, not propagate.
        let mut bad = scales.clone();
        bad[0] = 255;
        assert!(unpack_mxfp4(&packed, &bad, rows, cols / 2).is_err());
    }

    #[test]
    fn minicpm3_arch_scales_and_factors() {
        let cfg: serde_json::Value = serde_json::from_str(
            r#"{
            "model_type": "minicpm3",
            "hidden_size": 2560, "num_hidden_layers": 62,
            "num_attention_heads": 40, "intermediate_size": 6400,
            "vocab_size": 73448, "rms_norm_eps": 1e-5, "rope_theta": 10000.0,
            "q_lora_rank": 768, "kv_lora_rank": 256,
            "qk_rope_head_dim": 32, "qk_nope_head_dim": 64, "v_head_dim": 64,
            "scale_emb": 12, "scale_depth": 1.4, "dim_model_base": 256,
            "hidden_act": "silu",
            "max_position_embeddings": 32768,
            "rope_scaling": {
                "type": "longrope",
                "short_factor": [1.05, 1.12, 1.25, 1.53, 2.09, 3.14, 4.93, 7.52,
                                 10.47, 13.06, 14.85, 15.90, 16.46, 16.74, 16.87, 16.94],
                "long_factor":  [1.05, 1.12, 1.25, 1.53, 2.09, 3.14, 4.93, 7.52,
                                 10.47, 13.06, 14.85, 15.90, 16.46, 16.74, 16.87, 16.94],
                "original_max_position_embeddings": 32768
            }
        }"#,
        )
        .unwrap();
        let arch = build_arch(&cfg).unwrap();
        assert_eq!(arch.embed_multiplier, 12.0, "scale_emb");
        assert_eq!(arch.logit_multiplier, Some(0.1), "dim_model_base/hidden");
        let fac = arch
            .rope_freq_factors
            .as_ref()
            .expect("short factors carried");
        assert_eq!(fac.len(), 16);
        assert!((fac[15] - 16.94).abs() < 1e-9);
        assert!(arch.tie_word_embeddings, "no lm_head tensor — tied");
        let mla = arch.mla.as_ref().expect("MLA");
        assert_eq!(mla.q_lora_rank, Some(768), "compressed q");
        assert!(!mla.nope);
        assert_eq!(arch.max_position_embeddings, 32768, "native window");
    }

    #[test]
    fn kimi_canon_names() {
        let c = |s: &str| canon_name(s).unwrap();
        // K3 wrapper prefix + mixtral-style expert names → canonical MoE.
        assert_eq!(
            c("language_model.model.layers.3.block_sparse_moe.experts.5.w1.weight"),
            "model.layers.3.mlp.experts.5.gate_proj.weight"
        );
        assert_eq!(
            c("model.layers.2.block_sparse_moe.experts.0.w2.weight"),
            "model.layers.2.mlp.experts.0.down_proj.weight"
        );
        assert_eq!(
            c("model.layers.2.block_sparse_moe.experts.0.w3.weight"),
            "model.layers.2.mlp.experts.0.up_proj.weight"
        );
        assert_eq!(
            c("model.layers.2.block_sparse_moe.gate.weight"),
            "model.layers.2.mlp.gate.weight"
        );
        assert_eq!(
            c("model.layers.2.block_sparse_moe.gate.e_score_correction_bias"),
            "model.layers.2.mlp.expert_bias"
        );
        assert_eq!(
            c("model.layers.2.block_sparse_moe.shared_experts.gate_proj.weight"),
            "model.layers.2.mlp.shared_expert.gate_proj.weight"
        );
        // Vision tower dropped (text tower converts alone).
        assert_eq!(
            canon_name("vision_tower.encoder.blocks.0.wqkv.weight"),
            None
        );
        assert_eq!(canon_name("mm_projector.proj.0.weight"), None);
    }

    #[test]
    fn kimi_linear_arch_layers_and_geometry() {
        // Kimi-Linear-48B shape in miniature: 8 layers, full attention on
        // the 1-BASED layers [4, 8], KDA elsewhere; MLA NoPE; sigmoid MoE.
        let cfg: serde_json::Value = serde_json::from_str(
            r#"{
            "model_type": "kimi_linear",
            "hidden_size": 2304, "num_hidden_layers": 8,
            "num_attention_heads": 32, "num_key_value_heads": 32,
            "intermediate_size": 9216, "vocab_size": 1000,
            "rms_norm_eps": 1e-5, "rope_theta": 10000.0,
            "kv_lora_rank": 512, "qk_rope_head_dim": 64,
            "qk_nope_head_dim": 128, "v_head_dim": 128,
            "mla_use_nope": true,
            "hidden_act": "silu",
            "num_experts": 256, "num_experts_per_token": 8,
            "moe_intermediate_size": 1024, "num_shared_experts": 1,
            "moe_renormalize": true, "moe_router_activation_func": "sigmoid",
            "first_k_dense_replace": 1,
            "linear_attn_config": {
                "full_attn_layers": [4, 8],
                "kda_layers": [1, 2, 3, 5, 6, 7],
                "head_dim": 128, "num_heads": 32,
                "short_conv_kernel_size": 4
            }
        }"#,
        )
        .unwrap();
        let arch = build_arch(&cfg).unwrap();
        assert_eq!(arch.num_layers, 8);
        // 1-based full_attn_layers [4,8] → 0-based indices 3 and 7.
        for (i, t) in arch.layer_types.iter().enumerate() {
            let want_full = i == 3 || i == 7;
            assert_eq!(
                matches!(t, cortiq_core::LayerType::FullAttention),
                want_full,
                "layer {i}"
            );
            assert_eq!(
                matches!(t, cortiq_core::LayerType::Kda),
                !want_full,
                "layer {i}"
            );
        }
        assert_eq!(arch.linear_num_key_heads, Some(32));
        assert_eq!(arch.linear_key_head_dim, Some(128));
        assert_eq!(arch.linear_value_head_dim, Some(128));
        assert_eq!(arch.linear_conv_kernel_dim, Some(4));
        let mla = arch.mla.as_ref().expect("kv_lora_rank → MLA");
        assert!(mla.nope, "mla_use_nope must carry into the header");
        assert_eq!(mla.q_lora_rank, None);
        let moe = arch.moe.as_ref().expect("num_experts → MoE");
        assert_eq!(moe.num_experts, 256);
        assert_eq!(moe.top_k, 8);
        assert!(moe.router_sigmoid, "sigmoid router");
        assert!(moe.norm_topk_prob, "moe_renormalize");
        assert_eq!(moe.shared_expert_intermediate_size, Some(1024));
        assert!(arch.kda_gate_lower_bound.is_none());
    }

    #[test]
    fn kimi_k3_unsupported_features_bail_honestly() {
        let cfg: serde_json::Value = serde_json::from_str(
            r#"{
            "model_type": "kimi_k3",
            "text_config": {
                "model_type": "kimi_linear",
                "hidden_size": 7168, "num_hidden_layers": 4,
                "num_attention_heads": 96, "intermediate_size": 33792,
                "vocab_size": 1000, "rms_norm_eps": 1e-5,
                "kv_lora_rank": 512, "qk_rope_head_dim": 64,
                "qk_nope_head_dim": 128, "v_head_dim": 128,
                "attn_res_block_size": 12,
                "linear_attn_config": {
                    "full_attn_layers": [4],
                    "head_dim": 128, "num_heads": 96,
                    "short_conv_kernel_size": 4,
                    "gate_lower_bound": -5.0
                }
            }
        }"#,
        )
        .unwrap();
        let err = build_arch(&cfg).unwrap_err().to_string();
        assert!(err.contains("attn_res_block_size"), "got: {err}");
    }

    #[test]
    fn fused_gdn_split_is_correct_permutation() {
        // nk=2, dk=3, nv=4 (r=2), dv=2, hid=1. Each source row's value = its flat
        // row index, so we can trace exactly where each row lands after the split.
        let (nk, dk, nv, dv, hid) = (2usize, 3usize, 4usize, 2usize, 1usize);
        let r = nv / nk; // 2
        let gw = 2 * dk + 2 * r * dv; // 6 + 8 = 14
        let w: Vec<f32> = (0..nk * gw * hid).map(|i| i as f32).collect();
        let out =
            split_fused_gdn("m.linear_attn.in_proj_qkvz.weight", &w, hid, nk, dk, nv, dv).unwrap();
        let qkv = &out[0];
        assert_eq!(qkv.0, "m.linear_attn.in_proj_qkv.weight");
        assert_eq!(qkv.2, 2 * nk * dk + nv * dv); // 12 + 8 = 20
        // q rows: group g row rr -> source flat row g*gw+rr.
        // g=0 -> rows 0,1,2 ; g=1 -> rows 14,15,16
        assert_eq!(qkv.1[0..3], [0.0, 1.0, 2.0]);
        assert_eq!(qkv.1[3..6], [14.0, 15.0, 16.0]);
        // k rows: source g*gw + dk+rr. g=0 -> 3,4,5 ; g=1 -> 17,18,19
        assert_eq!(qkv.1[6..9], [3.0, 4.0, 5.0]);
        assert_eq!(qkv.1[9..12], [17.0, 18.0, 19.0]);
        // v rows: source g*gw + 2dk+rr (rr 0..4). g=0 -> 6,7,8,9 ; g=1 -> 20,21,22,23
        assert_eq!(qkv.1[12..16], [6.0, 7.0, 8.0, 9.0]);
        assert_eq!(qkv.1[16..20], [20.0, 21.0, 22.0, 23.0]);
        let z = &out[1];
        assert_eq!(z.0, "m.linear_attn.in_proj_z.weight");
        assert_eq!(z.2, nv * dv); // 8
        // z rows: source g*gw + 2dk+r*dv+rr. g=0 -> 10,11,12,13 ; g=1 -> 24,25,26,27
        assert_eq!(z.1, [10.0, 11.0, 12.0, 13.0, 24.0, 25.0, 26.0, 27.0]);

        // in_proj_ba: group width 2r=4. rows nk*4 = 8.
        let wb: Vec<f32> = (0..nk * 2 * r * hid).map(|i| i as f32).collect();
        let outb =
            split_fused_gdn("m.linear_attn.in_proj_ba.weight", &wb, hid, nk, dk, nv, dv).unwrap();
        // b = first r per group: g=0 -> 0,1 ; g=1 -> 4,5
        assert_eq!(outb[0].0, "m.linear_attn.in_proj_b.weight");
        assert_eq!(outb[0].1, [0.0, 1.0, 4.0, 5.0]);
        // a = next r per group: g=0 -> 2,3 ; g=1 -> 6,7
        assert_eq!(outb[1].0, "m.linear_attn.in_proj_a.weight");
        assert_eq!(outb[1].1, [2.0, 3.0, 6.0, 7.0]);
    }

    /// LFM2.5 states its mixer per layer in `layer_types`, spelling the
    /// recurrent one `conv`. The header has to carry that verbatim: a
    /// layer read as full attention would allocate a KV cache for a
    /// mixer that has none, and the short-conv ring would never advance.
    /// A VL wrapper's vision tower must not ride into the file: 333
    /// `model.visual.*` tensors under raw names is ~a gigabyte of dead
    /// weight the loader never reads (Ornith-1.5 / Qwen3.5-VL layout).
    #[test]
    fn a_vision_tower_is_skipped_whole() {
        for n in [
            "model.visual.patch_embed.proj.weight",
            "model.visual.blocks.0.attn.qkv.weight",
            "model.visual.merger.mlp.0.weight",
            "model.merger.mlp.0.linear_1.weight",
        ] {
            assert!(canon_name(n).is_none(), "{n} must be skipped");
        }
        // The text tower under the same wrapper still converts.
        assert_eq!(
            canon_name("model.language_model.layers.0.mlp.gate_proj.weight").as_deref(),
            Some("model.layers.0.mlp.gate_proj.weight")
        );
    }

    #[test]
    fn minicpmv47_wrapper_uses_qwen35_text_semantics() {
        // The public MiniCPM-V 4.7 config exposes the decoder only through
        // text_config.  Its wrapper must not lose Qwen3.5's zero-centered
        // RMSNorm or its renormalized top-k router contract.
        let cfg = serde_json::json!({
            "model_type": "minicpmv4_7",
            "tie_word_embeddings": false,
            "text_config": {
                "model_type": "qwen3_5_moe_text",
                "hidden_size": 64,
                "moe_intermediate_size": 16,
                "num_hidden_layers": 4,
                "num_attention_heads": 4,
                "num_key_value_heads": 1,
                "head_dim": 16,
                "vocab_size": 128,
                "max_position_embeddings": 262144,
                "mtp_num_hidden_layers": 1,
                "partial_rotary_factor": 0.25,
                "rope_parameters": {
                    "rope_type": "default",
                    "rope_theta": 10000000,
                    "partial_rotary_factor": 0.25
                },
                "num_experts": 8,
                "num_experts_per_tok": 2,
                "shared_expert_intermediate_size": 16,
                "linear_conv_kernel_dim": 4,
                "linear_num_key_heads": 4,
                "linear_num_value_heads": 4,
                "linear_key_head_dim": 16,
                "linear_value_head_dim": 16,
                "layer_types": [
                    "linear_attention", "linear_attention",
                    "linear_attention", "full_attention"
                ],
                "rms_norm_eps": 0.000001
            }
        });
        let arch = build_arch(&cfg).unwrap();
        assert_eq!(arch.arch_name, "minicpmv4_7");
        assert_eq!(arch.norm_style, NormStyle::Gemma);
        assert!(
            arch.mtp.is_none(),
            "the public wrapper ships no MTP tensors"
        );
        assert!(arch.moe.as_ref().unwrap().norm_topk_prob);
        assert_eq!(arch.rope_theta, 10_000_000.0);
        assert_eq!(
            arch.layer_types,
            [
                LayerType::LinearAttention,
                LayerType::LinearAttention,
                LayerType::LinearAttention,
                LayerType::FullAttention,
            ]
        );
        assert_eq!(
            profile_quant(
                &arch,
                Quant::Q4TiledP,
                "model.layers.0.mlp.experts.0.gate_proj.weight"
            ),
            Quant::Q4TiledP
        );
        for name in [
            "model.embed_tokens.weight",
            "lm_head.weight",
            "model.layers.0.linear_attn.in_proj_qkv.weight",
            "model.layers.3.self_attn.q_proj.weight",
        ] {
            assert_eq!(
                profile_quant(&arch, Quant::Q4TiledP, name),
                Quant::Q8_2f,
                "{name}"
            );
        }
        for name in [
            "model.layers.0.mlp.shared_expert.gate_proj.weight",
            "model.layers.0.mlp.shared_expert.up_proj.weight",
            "model.layers.0.mlp.shared_expert.down_proj.weight",
        ] {
            assert_eq!(
                profile_quant(&arch, Quant::Q4TiledP, name),
                Quant::Q4TiledP,
                "{name} must match the routed-expert codec for GPU MoE graphs"
            );
        }
        assert_eq!(
            profile_quant(
                &arch,
                Quant::Q8_2f,
                "model.layers.0.mlp.experts.0.gate_proj.weight"
            ),
            Quant::Q8_2f
        );
    }

    #[test]
    fn minicpmv47_1b_dense_wrapper_uses_qwen35_text_semantics() {
        // The 1B sibling uses the dense `qwen3_5_text` decoder.  It shares
        // the wrapper and zero-centered norm contract. Its q4tp profile
        // keeps only the quality-sensitive vocabulary boundaries in q8_2f;
        // the ordinary dense decoder body remains q4tp.
        let cfg = serde_json::json!({
            "model_type": "minicpmv4_7",
            "tie_word_embeddings": false,
            "text_config": {
                "model_type": "qwen3_5_text",
                "hidden_size": 64,
                "intermediate_size": 128,
                "num_hidden_layers": 4,
                "num_attention_heads": 4,
                "num_key_value_heads": 1,
                "head_dim": 16,
                "vocab_size": 128,
                "max_position_embeddings": 262144,
                "mtp_num_hidden_layers": 1,
                "partial_rotary_factor": 0.25,
                "rope_parameters": {
                    "rope_type": "default",
                    "rope_theta": 10000000,
                    "partial_rotary_factor": 0.25
                },
                "linear_conv_kernel_dim": 4,
                "linear_num_key_heads": 4,
                "linear_num_value_heads": 4,
                "linear_key_head_dim": 16,
                "linear_value_head_dim": 16,
                "layer_types": [
                    "linear_attention", "linear_attention",
                    "linear_attention", "full_attention"
                ],
                "rms_norm_eps": 0.000001
            }
        });
        let arch = build_arch(&cfg).unwrap();
        assert_eq!(arch.arch_name, "minicpmv4_7");
        assert_eq!(arch.norm_style, NormStyle::Gemma);
        assert!(
            arch.mtp.is_none(),
            "the public wrapper ships no MTP tensors"
        );
        assert!(arch.moe.is_none());
        assert_eq!(
            profile_quant(
                &arch,
                Quant::Q4TiledP,
                "model.layers.0.linear_attn.in_proj_qkv.weight"
            ),
            Quant::Q4TiledP
        );
        assert_eq!(
            profile_quant(
                &arch,
                Quant::Q4TiledP,
                "model.layers.3.self_attn.q_proj.weight"
            ),
            Quant::Q4TiledP
        );
        for name in ["model.embed_tokens.weight", "lm_head.weight"] {
            assert_eq!(
                profile_quant(&arch, Quant::Q4TiledP, name),
                Quant::Q8_2f,
                "{name} is a quality-sensitive vocabulary boundary"
            );
        }
        assert!(force_f16("model.layers.0.linear_attn.in_proj_a.weight"));
        assert!(force_f16("model.layers.0.mlp.gate.weight"));
    }

    #[test]
    fn lfm25_layer_types_and_conv_cache_reach_the_header() {
        let cfg: serde_json::Value = serde_json::from_str(
            r#"{
            "model_type": "lfm2", "architectures": ["Lfm2ForCausalLM"],
            "hidden_size": 1024, "num_hidden_layers": 6,
            "num_attention_heads": 16, "num_key_value_heads": 8,
            "intermediate_size": 2560, "vocab_size": 65536,
            "norm_eps": 1e-5, "conv_L_cache": 3, "conv_bias": false,
            "layer_types": ["conv","conv","full_attention","conv","full_attention","conv"],
            "tie_word_embeddings": true
        }"#,
        )
        .unwrap();
        let arch = build_arch(&cfg).unwrap();
        assert_eq!(arch.num_layers, 6);
        for (i, t) in arch.layer_types.iter().enumerate() {
            let want_full = i == 2 || i == 4;
            assert_eq!(
                matches!(t, cortiq_core::LayerType::FullAttention),
                want_full,
                "layer {i} attention kind"
            );
            assert_eq!(
                matches!(t, cortiq_core::LayerType::ShortConv),
                !want_full,
                "layer {i} mixer kind"
            );
        }
        // The ring is `conv_L_cache` deep; without it the conv reads
        // whatever the state buffer happened to hold.
        assert_eq!(arch.linear_conv_kernel_dim, Some(3));
        assert!(arch.moe.is_none(), "the dense build has no experts");
    }

    /// The 8B-A1B build is the same mixer schedule with a routed FFN.
    /// LFM2 routes with a sigmoid gate rather than a softmax over all
    /// experts, and reading that wrong changes which experts fire.
    #[test]
    fn lfm25_moe_routes_with_a_sigmoid_gate() {
        let cfg: serde_json::Value = serde_json::from_str(
            r#"{
            "model_type": "lfm2_moe", "architectures": ["Lfm2MoeForCausalLM"],
            "hidden_size": 2048, "num_hidden_layers": 4,
            "num_attention_heads": 32, "num_key_value_heads": 8,
            "intermediate_size": 7168, "vocab_size": 128000,
            "norm_eps": 1e-5, "conv_L_cache": 3,
            "layer_types": ["conv","conv","full_attention","conv"],
            "num_experts": 32, "num_experts_per_tok": 4,
            "moe_intermediate_size": 1792, "norm_topk_prob": true,
            "routed_scaling_factor": 1.0,
            "tie_word_embeddings": true
        }"#,
        )
        .unwrap();
        let arch = build_arch(&cfg).unwrap();
        let moe = arch.moe.as_ref().expect("num_experts → MoE");
        assert_eq!(moe.num_experts, 32);
        assert_eq!(moe.top_k, 4);
        assert_eq!(moe.moe_intermediate_size, 1792);
        assert!(moe.router_sigmoid, "lfm2 routes with a sigmoid gate");
        assert!(moe.norm_topk_prob);
        // A scale of exactly 1.0 is the no-op default and must not ride
        // in the header as if it were a correction.
        assert!(moe.routed_scaling_factor.is_none());
        assert_eq!(arch.linear_conv_kernel_dim, Some(3));
    }

    #[test]
    fn lfm2_names_map_to_canonical_layout() {
        // Conv (dense) layer 0.
        let c = |s: &str| canon_name(s).unwrap();
        assert_eq!(c("model.embedding_norm.weight"), "model.norm.weight");
        assert_eq!(
            c("model.layers.0.operator_norm.weight"),
            "model.layers.0.input_layernorm.weight"
        );
        assert_eq!(
            c("model.layers.0.ffn_norm.weight"),
            "model.layers.0.post_attention_layernorm.weight"
        );
        assert_eq!(
            c("model.layers.0.conv.in_proj.weight"),
            "model.layers.0.short_conv.in_proj.weight"
        );
        assert_eq!(
            c("model.layers.0.conv.conv.weight"),
            "model.layers.0.short_conv.conv.weight"
        );
        assert_eq!(
            c("model.layers.0.conv.out_proj.weight"),
            "model.layers.0.short_conv.out_proj.weight"
        );
        // Dense FFN: w1/w3/w2 → gate/up/down.
        assert_eq!(
            c("model.layers.0.feed_forward.w1.weight"),
            "model.layers.0.mlp.gate_proj.weight"
        );
        assert_eq!(
            c("model.layers.0.feed_forward.w3.weight"),
            "model.layers.0.mlp.up_proj.weight"
        );
        assert_eq!(
            c("model.layers.0.feed_forward.w2.weight"),
            "model.layers.0.mlp.down_proj.weight"
        );
        // Attention (full_attention layer 2): out_proj → o_proj, q/k layernorm.
        assert_eq!(
            c("model.layers.2.self_attn.out_proj.weight"),
            "model.layers.2.self_attn.o_proj.weight"
        );
        assert_eq!(
            c("model.layers.2.self_attn.q_layernorm.weight"),
            "model.layers.2.self_attn.q_norm.weight"
        );
        assert_eq!(
            c("model.layers.2.self_attn.k_layernorm.weight"),
            "model.layers.2.self_attn.k_norm.weight"
        );
        // MoE router / bias / experts.
        assert_eq!(
            c("model.layers.2.feed_forward.gate.weight"),
            "model.layers.2.mlp.gate.weight"
        );
        assert_eq!(
            c("model.layers.2.feed_forward.expert_bias"),
            "model.layers.2.mlp.expert_bias"
        );
        assert_eq!(
            c("model.layers.2.feed_forward.experts.7.w1.weight"),
            "model.layers.2.mlp.experts.7.gate_proj.weight"
        );
        assert_eq!(
            c("model.layers.2.feed_forward.experts.7.w2.weight"),
            "model.layers.2.mlp.experts.7.down_proj.weight"
        );
        // Q/K/V projections already canonical — must pass through untouched.
        assert_eq!(
            c("model.layers.2.self_attn.q_proj.weight"),
            "model.layers.2.self_attn.q_proj.weight"
        );
        // A Qwen tensor must be untouched by the LFM2 rewrite.
        assert_eq!(
            c("model.layers.3.mlp.gate_proj.weight"),
            "model.layers.3.mlp.gate_proj.weight"
        );
    }

    #[test]
    fn lfm2_moe_arch_routing_and_layers() {
        let cfg: serde_json::Value = serde_json::from_str(
            r#"{"model_type":"lfm2_moe","hidden_size":2048,"num_hidden_layers":4,
                "num_attention_heads":32,"num_key_value_heads":8,"intermediate_size":7168,
                "moe_intermediate_size":1792,"vocab_size":128000,"norm_eps":1e-5,
                "conv_L_cache":3,"num_experts":32,"num_experts_per_tok":4,
                "norm_topk_prob":true,"use_expert_bias":true,"routed_scaling_factor":1.0,
                "tie_word_embeddings":true,"rope_parameters":{"rope_theta":5000000},
                "layer_types":["conv","conv","full_attention","conv"]}"#,
        )
        .unwrap();
        let arch = build_arch(&cfg).unwrap();
        assert_eq!(arch.layer_types[0], LayerType::ShortConv);
        assert_eq!(arch.layer_types[2], LayerType::FullAttention);
        assert_eq!(arch.head_dim, 64);
        assert_eq!(arch.linear_conv_kernel_dim, Some(3));
        assert!((arch.rms_norm_eps - 1e-5).abs() < 1e-12);
        let moe = arch.moe.as_ref().unwrap();
        assert!(
            moe.router_sigmoid,
            "lfm2_moe must route with a sigmoid gate"
        );
        assert_eq!(moe.top_k, 4);
        assert!(moe.norm_topk_prob);
        // Scale 1.0 stores as None (no-op).
        assert_eq!(moe.routed_scaling_factor, None);
    }

    /// The safety invariant that lets `q1p` be an unconditional replacement
    /// for `q1` on models that ARE 1-bit-trained: when every group weight
    /// already sits on ±s, the carry stays zero and no sign flips, so the
    /// encoder is bit-identical to the plain sign quantizer. (For a NORMAL
    /// checkpoint the two differ — that difference is the training-free PTQ,
    /// judged by end-to-end PPL, not by any single closed-form proxy.)
    #[test]
    fn q1_ef_bit_identical_on_a_1bit_tensor() {
        let (rows, cols) = (4usize, 96usize);
        let onebit: Vec<f32> = (0..rows * cols)
            .map(|i| if (i * 7 + 3) % 5 < 2 { 0.25 } else { -0.25 })
            .collect();
        assert_eq!(
            encode_q1(&onebit, rows, cols),
            encode_q1_ef(&onebit, rows, cols),
            "error diffusion must be a no-op on a genuinely 1-bit tensor"
        );
    }

    /// Q1S roundtrip: kept outliers come back at f16 precision and the bulk
    /// decodes to the per-group ±s level. Guards the format the fold will
    /// populate.
    #[test]
    fn q1s_roundtrip_restores_outliers_and_binarizes_the_rest() {
        use cortiq_core::quant::dequant_q1s;
        let (rows, cols) = (2usize, 64usize);
        let mut vals: Vec<f32> = (0..rows * cols)
            .map(|i| (i as f32 * 0.017).sin() * 0.1)
            .collect();
        let spikes = [5usize, 40, 70, 120];
        for &i in &spikes {
            vals[i] = if i % 2 == 0 { 3.0 } else { -3.0 };
        }
        let keep = spikes.len() as f32 / (rows * cols) as f32;
        let bytes = encode_q1s(&vals, rows, cols, keep);
        let mut dec = vec![0f32; rows * cols];
        dequant_q1s(&bytes, &mut dec);
        for &i in &spikes {
            assert!(
                (dec[i] - vals[i]).abs() < 0.02,
                "outlier {i}: {} vs {}",
                dec[i],
                vals[i]
            );
        }
        for i in 0..rows * cols {
            if !spikes.contains(&i) {
                assert!(
                    dec[i].abs() < 2.0,
                    "bulk {i} should be a small ±s, got {}",
                    dec[i]
                );
            }
        }
    }

    #[test]
    fn parse_quant_variants() {
        for q in ["q8", "q8_row", "q8_2f", "q4", "q4_block", "f16"] {
            assert!(parse_quant(q).is_ok(), "{q}");
        }
        assert!(parse_quant("nope").is_err());
    }

    #[test]
    fn q8_row_roundtrips() {
        let (o, i) = (4usize, 64usize);
        let vals: Vec<f32> = (0..o * i).map(|k| (k as f32 * 0.017).sin() * 2.5).collect();
        let bytes = encode_q8_row(&vals, o, i);
        assert_eq!(bytes.len(), o * i + o * 2);
        let mut back = vec![0f32; o * i];
        dequant_q8_row(&bytes, o, i, &mut back);
        for (a, b) in vals.iter().zip(&back) {
            assert!((a - b).abs() < 0.05, "{a} vs {b}");
        }
    }

    #[test]
    fn q8_2f_roundtrips() {
        let (o, i) = (8usize, 48usize);
        let vals: Vec<f32> = (0..o * i).map(|k| (k as f32 * 0.023).cos() * 1.7).collect();
        let bytes = encode_q8_2f(&vals, o, i);
        assert_eq!(bytes.len(), o * i + o * 2 + i * 2);
        let mut back = vec![0f32; o * i];
        dequant_q8_2f(&bytes, o, i, &mut back);
        for (a, b) in vals.iter().zip(&back) {
            assert!((a - b).abs() < 0.1, "{a} vs {b}");
        }
    }

    #[test]
    fn q4_block_roundtrips() {
        let vals: Vec<f32> = (0..128).map(|k| (k as f32 * 0.05).sin()).collect();
        let bytes = encode_q4_block(&vals);
        let mut back = vec![0f32; 128];
        dequant_q4_block(&bytes, &mut back);
        for (a, b) in vals.iter().zip(&back) {
            assert!((a - b).abs() < 0.2, "{a} vs {b}");
        }
    }

    /// A raw safetensors blob from F32 tensors, for the end-to-end test.
    fn tiny_safetensors(tensors: &[(&str, Vec<usize>, Vec<f32>)]) -> Vec<u8> {
        let mut header = serde_json::Map::new();
        let mut data = Vec::new();
        for (name, shape, vals) in tensors {
            let start = data.len();
            for &v in vals {
                data.extend_from_slice(&v.to_le_bytes());
            }
            header.insert(
                name.to_string(),
                serde_json::json!({"dtype":"F32","shape":shape,"data_offsets":[start, data.len()]}),
            );
        }
        let hjson = serde_json::to_vec(&serde_json::Value::Object(header)).unwrap();
        let mut out = (hjson.len() as u64).to_le_bytes().to_vec();
        out.extend_from_slice(&hjson);
        out.extend_from_slice(&data);
        out
    }

    #[test]
    fn local_ready_rejects_partial_or_changed_source_before_mmap() {
        let root = std::env::temp_dir().join(format!(
            "cortiq-local-ready-boundary-test-{}",
            std::process::id()
        ));
        let _ = fs::remove_dir_all(&root);
        let source_dir = root.join("hf");
        let ready_dir = root.join("ready");
        fs::create_dir_all(&source_dir).unwrap();
        fs::create_dir_all(&ready_dir).unwrap();
        let filename = "model-00001-of-00001.safetensors";
        let source = source_dir.join(filename);
        let bytes = b"complete safetensors source";
        fs::write(&source, bytes).unwrap();
        let (_, sha256) = sha256_file(&source).unwrap();
        let expected = SourceFileExpectation {
            size: bytes.len() as u64,
            sha256,
        };
        let marker = ready_marker_path(&ready_dir, filename).unwrap();
        fs::write(
            source.with_file_name(format!("{filename}.aria2")),
            b"partial",
        )
        .unwrap();
        fs::write(
            &marker,
            serde_json::to_vec(&serde_json::json!({
                "filename": filename,
                "size": expected.size,
                "sha256": expected.sha256
            }))
            .unwrap(),
        )
        .unwrap();
        let err = validate_local_ready_source(&ready_dir, &source_dir, filename, &expected)
            .expect_err("a source with .aria2 must never be mmaped");
        assert!(err.to_string().contains("aria2"), "unexpected error: {err}");
        fs::remove_file(source.with_file_name(format!("{filename}.aria2"))).unwrap();

        let mut changed = bytes.to_vec();
        changed[0] ^= 0x01;
        fs::write(&source, changed).unwrap();
        let err = validate_local_ready_source(&ready_dir, &source_dir, filename, &expected)
            .expect_err("a changed source must fail the pinned SHA256 check");
        assert!(
            err.to_string().contains("SHA256") || err.to_string().contains("validation"),
            "unexpected error: {err}"
        );

        fs::write(&source, bytes).unwrap();
        assert_eq!(
            validate_local_ready_source(&ready_dir, &source_dir, filename, &expected).unwrap(),
            source
        );
        fs::remove_dir_all(&root).unwrap();
    }

    #[test]
    fn local_ready_stream_processes_certified_shards_and_preserves_source() {
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        unsafe {
            std::env::set_var(LOCAL_READY_POLL_MS_ENV, "0");
            std::env::remove_var(LOCAL_READY_DIR_ENV);
            std::env::remove_var(CONSUME_SOURCE_SHARDS_ENV);
        }
        let root = std::env::temp_dir().join(format!(
            "cortiq-local-ready-stream-test-{}",
            std::process::id()
        ));
        let _ = fs::remove_dir_all(&root);
        let source_dir = root.join("hf");
        let ready_dir = root.join("ready");
        fs::create_dir_all(&source_dir).unwrap();
        fs::create_dir_all(&ready_dir).unwrap();
        let first = "model-00001-of-00002.safetensors";
        let second = "model-00002-of-00002.safetensors";
        let shards = [
            (
                first,
                tiny_safetensors(&[("embed.weight", vec![2, 2], vec![1.0, 2.0, 3.0, 4.0])]),
            ),
            (
                second,
                tiny_safetensors(&[("norm.weight", vec![2], vec![1.0, 1.0])]),
            ),
        ];
        let mut manifest = serde_json::Map::new();
        for (name, bytes) in &shards {
            fs::write(source_dir.join(name), bytes).unwrap();
            let (size, sha256) = sha256_file(&source_dir.join(name)).unwrap();
            assert_eq!(size, bytes.len() as u64);
            manifest.insert(
                (*name).to_string(),
                serde_json::json!({"size": size, "sha256": sha256}),
            );
            fs::write(
                ready_dir.join(format!("{name}.json")),
                serde_json::json!({
                    "filename": name,
                    "size": size,
                    "sha256": sha256
                })
                .to_string(),
            )
            .unwrap();
        }
        fs::write(
            root.join("source-files.json"),
            serde_json::to_vec(&serde_json::Value::Object(manifest)).unwrap(),
        )
        .unwrap();
        fs::write(
            source_dir.join("config.json"),
            serde_json::json!({
                "model_type": "deepseek_v41",
                "text_config": {
                    "model_type": "deepseek_v41_text",
                    "hidden_size": 2,
                    "intermediate_size": 4,
                    "num_hidden_layers": 1,
                    "num_attention_heads": 1,
                    "num_key_value_heads": 1,
                    "head_dim": 2,
                    "vocab_size": 2,
                    "rms_norm_eps": 1e-6
                }
            })
            .to_string(),
        )
        .unwrap();
        fs::write(
            source_dir.join("model.safetensors.index.json"),
            serde_json::json!({
                "weight_map": {
                    "embed.weight": first,
                    "norm.weight": second
                }
            })
            .to_string(),
        )
        .unwrap();

        let output = root.join("ready.cmf");
        let mut last_progress = 0.0f32;
        unsafe {
            std::env::set_var(LOCAL_READY_DIR_ENV, &ready_dir);
        }
        run_convert_multi(
            source_dir.to_str().unwrap(),
            &[("q8".into(), output.to_str().unwrap().into())],
            None,
            None,
            None,
            false,
            |p| last_progress = p,
        )
        .unwrap();
        assert_eq!(last_progress, 1.0);
        assert!(source_dir.join(first).exists());
        assert!(source_dir.join(second).exists());
        let model = CmfModel::open(&output).unwrap();
        assert!(
            model.verify().is_empty(),
            "CMF verify: {:?}",
            model.verify()
        );
        unsafe {
            std::env::remove_var(LOCAL_READY_DIR_ENV);
            std::env::remove_var(LOCAL_READY_POLL_MS_ENV);
        }
        fs::remove_dir_all(&root).unwrap();
    }

    #[test]
    fn source_shard_priority_resume_matches_default_payloads() {
        // The priority path must remain a pure scheduling change: start with
        // the second shard already written and durably marked, remove that
        // source as the consuming pod would, then resume in [second, first]
        // order.  Comparing every final tensor's bytes against an ordinary
        // [first, second] conversion catches both duplicate and missing work.
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        unsafe {
            std::env::remove_var(SOURCE_SHARD_PRIORITY_ENV);
            std::env::remove_var(CONSUME_SOURCE_SHARDS_ENV);
            std::env::set_var(LOCAL_READY_POLL_MS_ENV, "0");
        }
        let root = std::env::temp_dir().join(format!(
            "cortiq-source-priority-resume-test-{}",
            std::process::id()
        ));
        let _ = fs::remove_dir_all(&root);
        let default_dir = root.join("default");
        let default_ready = root.join("default-ready");
        let priority_dir = root.join("priority");
        let priority_ready = root.join("priority-ready");
        let first = "model-00001-of-00002.safetensors";
        let second = "model-00002-of-00002.safetensors";
        let first_values = vec![1.0f32, 2.0, 3.0, 4.0];
        let second_values = vec![1.0f32, 1.0];

        let write_fixture = |dir: &Path, ready: &Path| {
            fs::create_dir_all(dir).unwrap();
            fs::create_dir_all(ready).unwrap();
            let shards = [
                (
                    first,
                    tiny_safetensors(&[("embed.weight", vec![2, 2], first_values.clone())]),
                ),
                (
                    second,
                    tiny_safetensors(&[("norm.weight", vec![2], second_values.clone())]),
                ),
            ];
            let mut manifest = serde_json::Map::new();
            for (name, bytes) in &shards {
                let source = dir.join(name);
                fs::write(&source, bytes).unwrap();
                let (size, sha256) = sha256_file(&source).unwrap();
                manifest.insert(
                    (*name).to_string(),
                    serde_json::json!({"size": size, "sha256": sha256}),
                );
                fs::write(
                    ready.join(format!("{name}.json")),
                    serde_json::json!({
                        "filename": name,
                        "size": size,
                        "sha256": sha256
                    })
                    .to_string(),
                )
                .unwrap();
            }
            fs::write(
                ready
                    .parent()
                    .expect("ready directory has a fixture parent")
                    .join("source-files.json"),
                serde_json::to_vec(&serde_json::Value::Object(manifest)).unwrap(),
            )
            .unwrap();
            fs::write(
                dir.join("config.json"),
                serde_json::json!({
                    "model_type": "deepseek_v41",
                    "text_config": {
                        "model_type": "deepseek_v41_text",
                        "hidden_size": 2,
                        "intermediate_size": 4,
                        "num_hidden_layers": 1,
                        "num_attention_heads": 1,
                        "num_key_value_heads": 1,
                        "head_dim": 2,
                        "vocab_size": 2,
                        "rms_norm_eps": 1e-6
                    }
                })
                .to_string(),
            )
            .unwrap();
            fs::write(
                dir.join("model.safetensors.index.json"),
                serde_json::json!({
                    "weight_map": {
                        "embed.weight": first,
                        "norm.weight": second
                    }
                })
                .to_string(),
            )
            .unwrap();
        };
        write_fixture(&default_dir, &default_ready);
        write_fixture(&priority_dir, &priority_ready);

        let default_output = root.join("default.cmf");
        unsafe {
            std::env::set_var(LOCAL_READY_DIR_ENV, &default_ready);
        }
        run_convert_multi(
            default_dir.to_str().unwrap(),
            &[("f16".into(), default_output.to_str().unwrap().into())],
            None,
            None,
            None,
            false,
            |_| {},
        )
        .unwrap();

        // Seed the resumed output with the second shard's exact f16 payload.
        // Its mark is durable before the source is removed, which is the
        // checkpoint contract exercised by the production consume path.
        let priority_output = root.join("priority.cmf");
        let priority_manifest = format!("{}.manifest", priority_output.display());
        let mut writer = cortiq_core::format::CmfStreamWriter::new(
            &priority_output,
            cortiq_core::format::CmfStreamWriter::head_reserve_for(8192, 96),
        )
        .unwrap()
        .with_manifest(&priority_manifest)
        .unwrap();
        writer
            .push(
                "model.norm.weight",
                TensorDtype::F16,
                &[2],
                &encode_f16(&second_values),
            )
            .unwrap();
        writer.mark(second).unwrap();
        drop(writer);
        fs::remove_file(priority_dir.join(second)).unwrap();

        unsafe {
            std::env::set_var(LOCAL_READY_DIR_ENV, &priority_ready);
            std::env::set_var(SOURCE_SHARD_PRIORITY_ENV, format!("{second},{first}"));
        }
        run_convert_multi(
            priority_dir.to_str().unwrap(),
            &[("f16".into(), priority_output.to_str().unwrap().into())],
            None,
            None,
            None,
            true,
            |_| {},
        )
        .unwrap();

        let default_model = CmfModel::open(&default_output).unwrap();
        let priority_model = CmfModel::open(&priority_output).unwrap();
        assert!(default_model.verify().is_empty());
        assert!(priority_model.verify().is_empty());
        assert_eq!(default_model.tensors.len(), priority_model.tensors.len());
        for expected in &default_model.tensors {
            let actual = priority_model
                .tensor(&expected.name)
                .unwrap_or_else(|| panic!("resumed output lacks {}", expected.name));
            assert_eq!(actual.dtype, expected.dtype, "dtype {}", expected.name);
            assert_eq!(actual.shape, expected.shape, "shape {}", expected.name);
            assert_eq!(actual.nbytes, expected.nbytes, "size {}", expected.name);
            assert_eq!(actual.hash, expected.hash, "hash {}", expected.name);
            assert_eq!(
                default_model.tensor_bytes(&expected.name).unwrap(),
                priority_model.tensor_bytes(&expected.name).unwrap(),
                "payload {}",
                expected.name
            );
        }
        assert!(priority_dir.join(first).exists());
        assert!(!priority_dir.join(second).exists());
        unsafe {
            std::env::remove_var(LOCAL_READY_DIR_ENV);
            std::env::remove_var(LOCAL_READY_POLL_MS_ENV);
            std::env::remove_var(SOURCE_SHARD_PRIORITY_ENV);
        }
        fs::remove_dir_all(&root).unwrap();
    }

    #[test]
    fn resume_skips_deleted_marked_shard_but_rejects_unmarked_missing_input() {
        // This is the failure mode of a V4.1 pod restart: the first shard was
        // durably marked and consumed before the process died, while the next
        // shard still has to be opened. The index and manifest are the only
        // files that survive source-shard consumption.
        let dir =
            std::env::temp_dir().join(format!("cortiq-resume-shard-test-{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        let first = "model-00001-of-00002.safetensors";
        let second = "model-00002-of-00002.safetensors";
        fs::write(
            dir.join(first),
            tiny_safetensors(&[("layers.0.weight", vec![2, 2], vec![1.0, 2.0, 3.0, 4.0])]),
        )
        .unwrap();
        fs::write(
            dir.join(second),
            tiny_safetensors(&[("layers.1.weight", vec![2, 2], vec![5.0, 6.0, 7.0, 8.0])]),
        )
        .unwrap();
        fs::write(
            dir.join("model.safetensors.index.json"),
            serde_json::to_vec(&serde_json::json!({
                "metadata": {},
                "weight_map": {
                    "layers.0.weight": first,
                    "layers.1.weight": second
                }
            }))
            .unwrap(),
        )
        .unwrap();

        let counts = source_shard_counts(&dir).unwrap();
        assert_eq!(
            counts,
            vec![(first.to_string(), 1), (second.to_string(), 1)]
        );
        let mut files = open_model(&dir).unwrap();
        assert_eq!(files.len(), 2);
        consume_source_shard(&mut files, 0, &dir).unwrap();
        assert!(!dir.join(first).exists());
        drop(files);

        let manifest = dir.join("resume.cmf.manifest");
        fs::write(
            &manifest,
            format!(
                "{{\"data_off\":4096}}\n{{\"mark\":\"{first}\",\"at\":16}}\n{{truncated\n{{\"mark\":\"{second}\",\"at\":32}}"
            ),
        )
        .unwrap();
        let marks = resume_manifest_marks(&manifest);
        assert!(marks.contains(first));
        assert!(!marks.contains(second));

        let pending = open_model_filtered(&dir, &marks).unwrap();
        assert_eq!(pending.len(), 1);
        assert_eq!(pending[0].name, second);
        assert_eq!(pending[0].tensors[0].name, "layers.1.weight");
        drop(pending);

        // A missing shard without a durable mark must remain a hard error;
        // silently skipping it would produce a valid-looking incomplete CMF.
        fs::remove_file(dir.join(second)).unwrap();
        assert!(open_model_filtered(&dir, &marks).is_err());
        fs::remove_dir_all(&dir).unwrap();
    }

    #[test]
    fn resume_after_checkpoint_consumes_deleted_inputs_and_finalizes_in_place() {
        // Exercise the complete local path with a partial CMF: its first
        // source shard is checkpointed, then deleted as if the pod died after
        // consumption. Resume must retain that durable payload, convert the
        // remaining shard, and finish the same output without a duplicate.
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        unsafe {
            std::env::set_var(CONSUME_SOURCE_SHARDS_ENV, "1");
        }
        let dir = std::env::temp_dir().join(format!(
            "cortiq-resume-in-place-test-{}",
            std::process::id()
        ));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        let first = "model-00001-of-00002.safetensors";
        let second = "model-00002-of-00002.safetensors";
        let first_values = vec![1.0f32, 2.0, 3.0, 4.0];
        fs::write(
            dir.join(first),
            tiny_safetensors(&[("embed.weight", vec![2, 2], first_values.clone())]),
        )
        .unwrap();
        fs::write(
            dir.join(second),
            tiny_safetensors(&[("norm.weight", vec![2], vec![1.0, 1.0])]),
        )
        .unwrap();
        let config = serde_json::json!({
            "model_type": "deepseek_v41",
            "text_config": {
                "model_type": "deepseek_v41_text",
                "hidden_size": 2,
                "intermediate_size": 4,
                "num_hidden_layers": 1,
                "num_attention_heads": 1,
                "num_key_value_heads": 1,
                "head_dim": 2,
                "vocab_size": 2,
                "rms_norm_eps": 1e-6
            }
        });
        fs::write(
            dir.join("config.json"),
            serde_json::to_vec(&config).unwrap(),
        )
        .unwrap();
        fs::write(
            dir.join("model.safetensors.index.json"),
            serde_json::to_vec(&serde_json::json!({
                "weight_map": {"embed.weight": first, "norm.weight": second}
            }))
            .unwrap(),
        )
        .unwrap();

        let output = dir.join("resume.cmf");
        let manifest = format!("{}.manifest", output.display());
        let mut writer = cortiq_core::format::CmfStreamWriter::new(
            &output,
            cortiq_core::format::CmfStreamWriter::head_reserve_for(8192, 96),
        )
        .unwrap()
        .with_manifest(&manifest)
        .unwrap();
        let first_bytes: Vec<u8> = first_values.iter().flat_map(|v| v.to_le_bytes()).collect();
        writer
            .push(
                "model.embed_tokens.weight",
                TensorDtype::F32,
                &[2, 2],
                &first_bytes,
            )
            .unwrap();
        writer.mark(first).unwrap();
        drop(writer);
        fs::remove_file(dir.join(first)).unwrap();

        let mut last_progress = 0.0f32;
        run_convert_multi(
            dir.to_str().unwrap(),
            &[("q8".into(), output.to_str().unwrap().into())],
            None,
            None,
            None,
            true,
            |p| last_progress = p,
        )
        .unwrap();
        assert_eq!(last_progress, 1.0);
        assert!(!dir.join(first).exists());
        assert!(!dir.join(second).exists());
        assert!(!std::path::Path::new(&manifest).exists());
        let model = CmfModel::open(&output).unwrap();
        assert!(
            model.verify().is_empty(),
            "CMF verify: {:?}",
            model.verify()
        );
        assert_eq!(
            model.arch().deepseek_v41.as_ref(),
            Some(&config),
            "the original V4.1 config must survive resume/finalization"
        );
        assert_eq!(
            model
                .tensors
                .iter()
                .filter(|t| t.name == "model.embed_tokens.weight")
                .count(),
            1,
            "resuming must not append the checkpointed shard twice"
        );
        unsafe {
            std::env::remove_var(CONSUME_SOURCE_SHARDS_ENV);
        }
        fs::remove_dir_all(&dir).unwrap();
    }

    /// `--resume` keeps the `--exclude-tensors` patterns the output was
    /// started with: started without them and resumed with a pattern that
    /// matches a tensor already written, or started with them and resumed
    /// without (or with others), is refused before anything is written; the
    /// same patterns finish the file without the excluded tensor, and the
    /// record goes with the manifest.
    #[test]
    fn resume_refuses_a_changed_tensor_exclusion() {
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        unsafe {
            std::env::remove_var(LOCAL_READY_DIR_ENV);
            std::env::remove_var(SOURCE_SHARD_PRIORITY_ENV);
            std::env::remove_var(CONSUME_SOURCE_SHARDS_ENV);
            std::env::remove_var(CONVERT_ONLY_ENV);
        }
        let dir = std::env::temp_dir().join(format!(
            "cortiq-resume-exclude-test-{}",
            std::process::id()
        ));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        let first = "model-00001-of-00002.safetensors";
        let second = "model-00002-of-00002.safetensors";
        let first_values = vec![1.0f32, 2.0, 3.0, 4.0];
        fs::write(
            dir.join(first),
            tiny_safetensors(&[("embed.weight", vec![2, 2], first_values.clone())]),
        )
        .unwrap();
        fs::write(
            dir.join(second),
            tiny_safetensors(&[("norm.weight", vec![2], vec![1.0, 1.0])]),
        )
        .unwrap();
        fs::write(
            dir.join("config.json"),
            serde_json::to_vec(&serde_json::json!({
                "model_type": "deepseek_v41",
                "text_config": {
                    "model_type": "deepseek_v41_text", "hidden_size": 2,
                    "intermediate_size": 4, "num_hidden_layers": 1,
                    "num_attention_heads": 1, "num_key_value_heads": 1, "head_dim": 2,
                    "vocab_size": 2, "rms_norm_eps": 1e-6
                }
            }))
            .unwrap(),
        )
        .unwrap();
        fs::write(
            dir.join("model.safetensors.index.json"),
            serde_json::to_vec(&serde_json::json!({
                "weight_map": {"embed.weight": first, "norm.weight": second}
            }))
            .unwrap(),
        )
        .unwrap();
        let output = dir.join("resume.cmf");
        let manifest = format!("{}.manifest", output.display());
        let record = exclusion_record_path(&manifest);
        // An interrupted run: the first shard written and marked.
        let partial = |record_patterns: Option<&[&str]>| {
            let _ = fs::remove_file(&output);
            let _ = fs::remove_file(&manifest);
            let _ = fs::remove_file(&record);
            let mut w = cortiq_core::format::CmfStreamWriter::new(
                &output,
                cortiq_core::format::CmfStreamWriter::head_reserve_for(8192, 96),
            )
            .unwrap()
            .with_manifest(&manifest)
            .unwrap();
            let bytes: Vec<u8> = first_values.iter().flat_map(|v| v.to_le_bytes()).collect();
            w.push("model.embed_tokens.weight", TensorDtype::F32, &[2, 2], &bytes)
                .unwrap();
            w.mark(first).unwrap();
            drop(w);
            if let Some(p) = record_patterns {
                fs::write(&record, serde_json::to_vec(p).unwrap()).unwrap();
            }
        };
        let resume = |exclude: &[&str]| {
            let exclude: Vec<String> = exclude.iter().map(|s| s.to_string()).collect();
            run_convert_multi_excluding(
                dir.to_str().unwrap(),
                &[("q8".into(), output.to_str().unwrap().into())],
                None,
                None,
                None,
                true,
                MimoTowers::TextOnly,
                &exclude,
                |_| {},
            )
        };

        partial(None);
        let e = resume(&["model.embed_tokens.weight"]).unwrap_err().to_string();
        assert!(e.contains("started without --exclude-tensors"), "{e}");
        assert!(e.contains("model.embed_tokens.weight"), "{e}");

        partial(Some(&["model.norm.weight"]));
        let e = resume(&[]).unwrap_err().to_string();
        assert!(
            e.contains("started with --exclude-tensors 'model.norm.weight' and is resumed with none"),
            "{e}"
        );
        let e = resume(&["model.norm.*"]).unwrap_err().to_string();
        assert!(e.contains("resumed with 'model.norm.*'"), "{e}");

        partial(Some(&["model.norm.weight"]));
        resume(&["model.norm.weight"]).unwrap();
        let model = CmfModel::open(&output).unwrap();
        assert!(model.verify().is_empty(), "{:?}", model.verify());
        let names: Vec<&str> = model.tensors.iter().map(|t| t.name.as_str()).collect();
        assert!(names.contains(&"model.embed_tokens.weight"), "{names:?}");
        assert!(!names.contains(&"model.norm.weight"), "{names:?}");
        assert_eq!(
            model.header.provenance.as_ref().unwrap()["excluded_tensors"],
            serde_json::json!(["model.norm.weight"])
        );
        drop(model);
        assert!(!std::path::Path::new(&manifest).exists());
        assert!(!std::path::Path::new(&record).exists());

        // Started without patterns, nothing excluded written yet: the resume
        // may add them (the record is written for the next resume).
        partial(None);
        resume(&["model.norm.weight"]).unwrap();
        let model = CmfModel::open(&output).unwrap();
        assert!(!model.tensors.iter().any(|t| t.name == "model.norm.weight"));
        drop(model);
        fs::remove_dir_all(&dir).unwrap();
    }

    #[test]
    fn convert_tiny_model_end_to_end() {
        // run_convert reads CMF_CONVERT_ONLY and the resume knobs, which
        // other tests set under ENV_LOCK: without it this conversion could
        // come out a partial probe file.
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        let dir = std::env::temp_dir().join(format!("cortiq-convtest-{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        fs::write(
            dir.join("config.json"),
            r#"{"model_type":"llama","hidden_size":64,"num_hidden_layers":1,"num_attention_heads":4,"num_key_value_heads":4,"intermediate_size":128,"vocab_size":32,"rms_norm_eps":0.000001,"tie_word_embeddings":true}"#,
        )
        .unwrap();
        fs::write(dir.join("tokenizer.json"), b"{}").unwrap();
        let st = tiny_safetensors(&[
            (
                "model.embed_tokens.weight",
                vec![32, 64],
                (0..32 * 64).map(|k| (k as f32 * 0.01).sin()).collect(),
            ),
            ("model.norm.weight", vec![64], vec![1.0f32; 64]),
        ]);
        fs::write(dir.join("model.safetensors"), &st).unwrap();
        let out = dir.join("m.cmf");
        run_convert(
            dir.to_str().unwrap(),
            "q8",
            out.to_str().unwrap(),
            None,
            None,
            None,
            false,
            |_| {},
        )
        .unwrap();

        let model = CmfModel::open(&out).unwrap();
        assert_eq!(model.arch().vocab_size, 32);
        assert_eq!(model.arch().num_layers, 1);
        let _ = fs::remove_dir_all(&dir);
    }

    #[test]
    fn tiny_mellum_auto_converts_loads_and_executes_its_dual_rope_schedule() {
        // A generated, fully-shaped Mellum fixture is intentionally used
        // instead of upstream weights.  It crosses the converter, CMF
        // directory, generic MoE loader and one real forward pass, including
        // all three local-attention layers and the YaRN full-attention layer.
        // ENV_LOCK: run_convert reads CMF_CONVERT_ONLY, which the probe test
        // sets for its own conversion.
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        let dir = std::env::temp_dir().join(format!(
            "cortiq-mellum-convtest-{}",
            std::process::id()
        ));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        let config = serde_json::json!({
            "model_type": "mellum",
            "hidden_size": 32,
            "intermediate_size": 64,
            "num_hidden_layers": 4,
            "num_attention_heads": 4,
            "num_key_value_heads": 1,
            "head_dim": 8,
            "vocab_size": 32,
            "max_position_embeddings": 131072,
            "rms_norm_eps": 0.000001,
            "hidden_act": "silu",
            "tie_word_embeddings": false,
            "sliding_window": 1024,
            "layer_types": [
                "sliding_attention", "sliding_attention",
                "sliding_attention", "full_attention"
            ],
            "num_experts": 2,
            "num_experts_per_tok": 1,
            "moe_intermediate_size": 32,
            "norm_topk_prob": true,
            "rope_parameters": {
                "full_attention": {
                    "rope_type": "yarn",
                    "rope_theta": 500000.0,
                    "factor": 16.0,
                    "original_max_position_embeddings": 8192,
                    "beta_fast": 32.0,
                    "beta_slow": 1.0,
                    "attention_factor": 1.2772588722239782
                },
                "sliding_attention": {
                    "rope_type": "default",
                    "rope_theta": 500000.0
                }
            }
        });
        fs::write(
            dir.join("config.json"),
            serde_json::to_vec(&config).unwrap(),
        )
        .unwrap();
        // Valid minimal HF tokenizer JSON.  The test drives `forward_ids`,
        // but embedding this makes the runnable CMF's tokenizer path real.
        fs::write(
            dir.join("tokenizer.json"),
            r#"{"model":{"type":"BPE","vocab":{"a":0,"b":1},"merges":[]}}"#,
        )
        .unwrap();

        let values = |n: usize, salt: usize| {
            (0..n)
                .map(|i| ((i + salt) as f32 * 0.017).sin() * 0.05)
                .collect::<Vec<_>>()
        };
        let mut tensors: Vec<(String, Vec<usize>, Vec<f32>)> = vec![
            (
                "model.embed_tokens.weight".into(),
                vec![32, 32],
                values(32 * 32, 1),
            ),
            ("lm_head.weight".into(), vec![32, 32], values(32 * 32, 2)),
            ("model.norm.weight".into(), vec![32], vec![1.0; 32]),
        ];
        for li in 0..4 {
            let p = format!("model.layers.{li}.");
            tensors.extend([
                (
                    format!("{p}input_layernorm.weight"),
                    vec![32],
                    vec![1.0; 32],
                ),
                (
                    format!("{p}post_attention_layernorm.weight"),
                    vec![32],
                    vec![1.0; 32],
                ),
                (
                    format!("{p}self_attn.q_proj.weight"),
                    vec![32, 32],
                    values(32 * 32, 10 + li),
                ),
                (
                    format!("{p}self_attn.k_proj.weight"),
                    vec![8, 32],
                    values(8 * 32, 20 + li),
                ),
                (
                    format!("{p}self_attn.v_proj.weight"),
                    vec![8, 32],
                    values(8 * 32, 30 + li),
                ),
                (
                    format!("{p}self_attn.o_proj.weight"),
                    vec![32, 32],
                    values(32 * 32, 40 + li),
                ),
                (
                    format!("{p}self_attn.q_norm.weight"),
                    vec![8],
                    vec![1.0; 8],
                ),
                (
                    format!("{p}self_attn.k_norm.weight"),
                    vec![8],
                    vec![1.0; 8],
                ),
                (
                    format!("{p}mlp.gate.weight"),
                    vec![2, 32],
                    values(2 * 32, 50 + li),
                ),
            ]);
            for expert in 0..2 {
                let ep = format!("{p}mlp.experts.{expert}.");
                tensors.extend([
                    (
                        format!("{ep}gate_proj.weight"),
                        vec![32, 32],
                        values(32 * 32, 60 + li * 2 + expert),
                    ),
                    (
                        format!("{ep}up_proj.weight"),
                        vec![32, 32],
                        values(32 * 32, 70 + li * 2 + expert),
                    ),
                    (
                        format!("{ep}down_proj.weight"),
                        vec![32, 32],
                        values(32 * 32, 80 + li * 2 + expert),
                    ),
                ]);
            }
        }
        let refs: Vec<(&str, Vec<usize>, Vec<f32>)> = tensors
            .iter()
            .map(|(name, shape, vals)| (name.as_str(), shape.clone(), vals.clone()))
            .collect();
        fs::write(dir.join("model.safetensors"), tiny_safetensors(&refs)).unwrap();
        let out = dir.join("tiny-mellum.cmf");
        run_convert(
            dir.to_str().unwrap(),
            AUTO_QUANT,
            out.to_str().unwrap(),
            None,
            None,
            None,
            false,
            |_| {},
        )
        .unwrap();

        let model = std::sync::Arc::new(CmfModel::open(&out).unwrap());
        assert!(model.verify().is_empty(), "CMF verify: {:?}", model.verify());
        assert_eq!(model.arch().sliding_window, Some(1024));
        assert_eq!(
            model.arch().yarn.as_ref().unwrap().attention_factor,
            1.2772588722239782
        );
        // The Mellum `auto` profile is intentionally mixed: its sparse
        // expert bank is q4tp, the always-active attention/vocabulary
        // skeleton is q8_2f, and the router/norms remain f16.
        let dtype = |name: &str| model.tensor(name).unwrap().dtype;
        assert_eq!(dtype("model.embed_tokens.weight"), TensorDtype::Q8_2f);
        assert_eq!(dtype("lm_head.weight"), TensorDtype::Q8_2f);
        assert_eq!(
            dtype("model.layers.0.self_attn.q_proj.weight"),
            TensorDtype::Q8_2f
        );
        assert_eq!(
            dtype("model.layers.0.mlp.experts.0.gate_proj.weight"),
            TensorDtype::Q4TiledP
        );
        assert_eq!(
            dtype("model.layers.0.mlp.experts.0.up_proj.weight"),
            TensorDtype::Q4TiledP
        );
        assert_eq!(
            dtype("model.layers.0.mlp.experts.0.down_proj.weight"),
            TensorDtype::Q4TiledP
        );
        assert_eq!(dtype("model.layers.0.mlp.gate.weight"), TensorDtype::F16);
        assert_eq!(
            dtype("model.layers.0.input_layernorm.weight"),
            TensorDtype::F16
        );
        let mut pipeline = cortiq_engine::Pipeline::from_model(
            &model,
            cortiq_engine::sampler::SamplerConfig::default(),
        )
        .unwrap();
        assert_eq!(pipeline.swa, Some((1024, usize::MAX)));
        assert_eq!(
            pipeline.sliding_layers,
            Some(vec![true, true, true, false])
        );
        assert_eq!(pipeline.rope_base, 500000.0);
        assert_eq!(pipeline.rope_scale, 1.2772588722239782);
        assert_eq!(pipeline.rope_scale_local, 1.0);
        let full_yarn = cortiq_engine::attention::yarn_inv_freq(8, 500000.0, 16.0, 8192, 32.0, 1.0);
        let local_rope = cortiq_engine::attention::rope_inv_freq(8, 500000.0);
        assert_eq!(pipeline.rope_inv_freq(), full_yarn.as_slice());
        assert_eq!(pipeline.inv_freq_local.as_ref().unwrap().as_slice(), local_rope.as_slice());
        assert_ne!(full_yarn, local_rope, "full YaRN must not use the local profile");
        assert_eq!(pipeline.rotary_dim_local, Some(8));
        let logits = pipeline.forward_ids(&[0, 1], None).unwrap();
        assert_eq!(logits.len(), 32);
        assert!(logits.iter().all(|v| v.is_finite()));

        drop(pipeline);
        drop(model);
        let _ = fs::remove_dir_all(&dir);
    }

    #[test]
    fn tensor_quant_spec_and_glob() {
        assert!(glob_match("lm_head.weight", "lm_head.weight"));
        assert!(!glob_match("lm_head.weight", "lm_head.weight2"));
        assert!(glob_match(
            "model.layers.*.mlp.down_proj.weight",
            "model.layers.41.mlp.down_proj.weight"
        ));
        assert!(!glob_match(
            "model.layers.*.mlp.down_proj.weight",
            "model.layers.41.mlp.up_proj.weight"
        ));
        assert!(glob_match("*v_proj*", "model.layers.3.self_attn.v_proj.weight"));
        assert!(glob_match("model.layers.0.*", "model.layers.0.self_attn.q_proj.weight"));
        assert!(!glob_match("model.layers.0.*", "model.layers.10.self_attn.q_proj.weight"));
        assert!(glob_match("*", "anything"));
        let (p, q) = parse_tensor_quant("model.embed_tokens.weight=q8_2f").unwrap();
        assert_eq!((p.as_str(), q), ("model.embed_tokens.weight", Quant::Q8_2f));
        assert_eq!(parse_tensor_quant("x=F16").unwrap().1, Quant::F16);
        assert!(parse_tensor_quant("no-equals").is_err());
        assert!(parse_tensor_quant("=q8").is_err());
        assert!(parse_tensor_quant("x=q9").is_err());
    }

    /// `--tensor-quant` reaches the directory: the named tensors change
    /// dtype, everything else keeps the profile's, and the file loads.
    /// The overrides are process-global, so the fixture uses names (layer 1)
    /// that no other converting test writes.
    #[test]
    fn tensor_quant_override_changes_only_matching_tensors() {
        // The overrides are process-global and run_convert reads the
        // local-ready env, which the resume tests set under ENV_LOCK.
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        unsafe {
            std::env::remove_var(LOCAL_READY_DIR_ENV);
            std::env::remove_var(SOURCE_SHARD_PRIORITY_ENV);
            std::env::remove_var(CONSUME_SOURCE_SHARDS_ENV);
        }
        let dir = std::env::temp_dir().join(format!("cortiq-tqtest-{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        fs::write(
            dir.join("config.json"),
            r#"{"model_type":"llama","hidden_size":64,"num_hidden_layers":2,"num_attention_heads":4,"num_key_value_heads":4,"intermediate_size":128,"vocab_size":32,"rms_norm_eps":0.000001,"tie_word_embeddings":true}"#,
        )
        .unwrap();
        fs::write(dir.join("tokenizer.json"), b"{}").unwrap();
        let vals = |n: usize| (0..n).map(|k| (k as f32 * 0.017).sin()).collect::<Vec<_>>();
        let mut tensors = vec![("model.embed_tokens.weight".to_string(), vec![32, 64], vals(32 * 64))];
        for li in 0..2 {
            for (t, shape) in [
                ("mlp.gate_proj", vec![128, 64]),
                ("mlp.up_proj", vec![128, 64]),
                ("mlp.down_proj", vec![64, 128]),
            ] {
                let n = shape[0] * shape[1];
                tensors.push((format!("model.layers.{li}.{t}.weight"), shape, vals(n)));
            }
        }
        tensors.push(("model.norm.weight".to_string(), vec![64], vec![1.0f32; 64]));
        let refs: Vec<(&str, Vec<usize>, Vec<f32>)> = tensors
            .iter()
            .map(|(n, s, v)| (n.as_str(), s.clone(), v.clone()))
            .collect();
        fs::write(dir.join("model.safetensors"), tiny_safetensors(&refs)).unwrap();
        let out = dir.join("mixed.cmf");
        set_tensor_quant_overrides(&[
            "model.layers.1.mlp.down_proj.weight=q8_2f".into(),
            "model.layers.1.mlp.up_*=f16".into(),
        ])
        .unwrap();
        let r = run_convert(
            dir.to_str().unwrap(),
            "q4tp",
            out.to_str().unwrap(),
            None,
            None,
            None,
            false,
            |_| {},
        );
        set_tensor_quant_overrides(&[]).unwrap();
        r.unwrap();
        let model = CmfModel::open(&out).unwrap();
        let dt = |n: &str| model.tensors.iter().find(|e| e.name == n).unwrap().dtype;
        assert_eq!(dt("model.layers.1.mlp.down_proj.weight"), TensorDtype::Q8_2f);
        assert_eq!(dt("model.layers.1.mlp.up_proj.weight"), TensorDtype::F16);
        assert_eq!(dt("model.layers.1.mlp.gate_proj.weight"), TensorDtype::Q4TiledP);
        assert_eq!(dt("model.layers.0.mlp.down_proj.weight"), TensorDtype::Q4TiledP);
        assert_eq!(dt("model.layers.0.mlp.up_proj.weight"), TensorDtype::Q4TiledP);
        let _ = fs::remove_dir_all(&dir);
    }

    #[test]
    fn multi_quant_emits_independent_profiles_and_matches_single_runs() {
        // This end-to-end fixture uses the ordinary local source path.  The
        // local-ready tests steer process-global environment variables while
        // they certify their temporary shards, so only this small group of
        // tests shares ENV_LOCK; the rest of the suite remains parallel.
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        unsafe {
            std::env::remove_var(LOCAL_READY_DIR_ENV);
            std::env::remove_var(LOCAL_READY_POLL_MS_ENV);
            std::env::remove_var(CONSUME_SOURCE_SHARDS_ENV);
            std::env::remove_var(SOURCE_SHARD_PRIORITY_ENV);
        }
        let dir =
            std::env::temp_dir().join(format!("cortiq-multi-convtest-{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        fs::write(
            dir.join("config.json"),
            r#"{"model_type":"llama","hidden_size":64,"num_hidden_layers":1,"num_attention_heads":4,"num_key_value_heads":4,"intermediate_size":128,"vocab_size":32,"rms_norm_eps":0.000001,"tie_word_embeddings":true}"#,
        )
        .unwrap();
        fs::write(dir.join("tokenizer.json"), b"{}").unwrap();
        let vals = |n: usize| (0..n).map(|k| (k as f32 * 0.013).sin()).collect::<Vec<_>>();
        let st = tiny_safetensors(&[
            ("model.embed_tokens.weight", vec![32, 64], vals(32 * 64)),
            (
                "model.layers.0.mlp.gate_proj.weight",
                vec![128, 64],
                vals(128 * 64),
            ),
            (
                "model.layers.0.mlp.up_proj.weight",
                vec![128, 64],
                vals(128 * 64),
            ),
            (
                "model.layers.0.mlp.down_proj.weight",
                vec![64, 128],
                vals(64 * 128),
            ),
            ("model.norm.weight", vec![64], vec![1.0f32; 64]),
        ]);
        fs::write(dir.join("model.safetensors"), &st).unwrap();

        let q4 = dir.join("single-q4.cmf");
        let q2 = dir.join("single-q2.cmf");
        run_convert(
            dir.to_str().unwrap(),
            "q4tp",
            q4.to_str().unwrap(),
            None,
            None,
            None,
            false,
            |_| {},
        )
        .unwrap();
        run_convert(
            dir.to_str().unwrap(),
            "q2tp",
            q2.to_str().unwrap(),
            None,
            None,
            None,
            false,
            |_| {},
        )
        .unwrap();
        let both4 = dir.join("both-q4.cmf");
        let both2 = dir.join("both-q2.cmf");
        run_convert_multi(
            dir.to_str().unwrap(),
            &[
                ("q4tp".into(), both4.to_str().unwrap().into()),
                ("q2tp".into(), both2.to_str().unwrap().into()),
            ],
            None,
            None,
            None,
            false,
            |_| {},
        )
        .unwrap();

        let single4 = CmfModel::open(&q4).unwrap();
        let single2 = CmfModel::open(&q2).unwrap();
        let multi4 = CmfModel::open(&both4).unwrap();
        let multi2 = CmfModel::open(&both2).unwrap();
        assert_eq!(single4.tensors.len(), multi4.tensors.len());
        assert_eq!(single2.tensors.len(), multi2.tensors.len());
        for (a, b) in single4.tensors.iter().zip(&multi4.tensors) {
            assert_eq!(
                (a.name.as_str(), a.dtype, a.shape.as_slice()),
                (b.name.as_str(), b.dtype, b.shape.as_slice())
            );
            assert_eq!(single4.entry_bytes(a), multi4.entry_bytes(b));
        }
        for (a, b) in single2.tensors.iter().zip(&multi2.tensors) {
            assert_eq!(
                (a.name.as_str(), a.dtype, a.shape.as_slice()),
                (b.name.as_str(), b.dtype, b.shape.as_slice())
            );
            assert_eq!(single2.entry_bytes(a), multi2.entry_bytes(b));
        }
        let gate4 = multi4
            .tensor("model.layers.0.mlp.gate_proj.weight")
            .unwrap();
        let gate2 = multi2
            .tensor("model.layers.0.mlp.gate_proj.weight")
            .unwrap();
        assert_eq!(gate4.dtype, TensorDtype::Q4TiledP);
        assert_eq!(gate2.dtype, TensorDtype::Q2TiledP);
        assert!(!both4.with_extension("cmf.manifest").exists());
        assert!(!both2.with_extension("cmf.manifest").exists());
        let _ = fs::remove_dir_all(&dir);
    }

    #[test]
    fn multi_quant_rejects_duplicate_outputs() {
        let err = run_convert_multi(
            "missing/model",
            &[
                ("q4tp".into(), "same.cmf".into()),
                ("q2tp".into(), "same.cmf".into()),
            ],
            None,
            None,
            None,
            false,
            |_| {},
        )
        .unwrap_err();
        assert!(err.to_string().contains("repeated"));
    }

    /// Qwen3.6 ships an MTP head the converter used to drop on the floor.
    /// It must survive, and land under the names the loader reads.
    #[test]
    fn mtp_head_is_kept_and_renamed_to_the_loader_layout() {
        let m = |s: &str| canon_name(s);
        assert_eq!(
            m("mtp.fc.weight").as_deref(),
            Some("model.mtp.eh_proj.weight")
        );
        assert_eq!(
            m("mtp.pre_fc_norm_embedding.weight").as_deref(),
            Some("model.mtp.enorm.weight")
        );
        assert_eq!(
            m("mtp.pre_fc_norm_hidden.weight").as_deref(),
            Some("model.mtp.hnorm.weight")
        );
        assert_eq!(
            m("mtp.norm.weight").as_deref(),
            Some("model.mtp.norm.weight")
        );
        // The block's own tensors pass through untouched — including the MoE
        // mlp, which is what this head actually carries.
        assert_eq!(
            m("mtp.layers.0.self_attn.q_proj.weight").as_deref(),
            Some("model.mtp.layers.0.self_attn.q_proj.weight")
        );
        assert_eq!(
            m("mtp.layers.0.mlp.experts.gate_up_proj").as_deref(),
            Some("model.mtp.layers.0.mlp.experts.gate_up_proj")
        );
        assert_eq!(
            m("mtp.layers.0.mlp.shared_expert_gate.weight").as_deref(),
            Some("model.mtp.layers.0.mlp.shared_expert_gate.weight")
        );
        // Vision towers are still dropped.
        assert!(m("visual.blocks.0.attn.qkv.weight").is_none());
    }

    #[test]
    fn prism_patch_kernel_is_flattened_in_processor_channel_order() {
        // Source layout is [out, temporal, height, width, channel]; the
        // runtime's flattened patch rows are [channel, temporal, height,
        // width].  Use one output row so every source coordinate is visible.
        let shape = [1, 2, 2, 2, 3];
        let source: Vec<u16> = (0..shape.iter().product::<usize>())
            .map(|v| f32_to_f16(v as f32).to_le())
            .collect();
        let raw = source
            .iter()
            .flat_map(|v| v.to_le_bytes())
            .collect::<Vec<_>>();
        let (flat_shape, flat) = flatten_prism_patch_weight("F16", &shape, &raw).unwrap();
        assert_eq!(flat_shape, vec![1, 24]);
        let mut want = Vec::new();
        for c in 0..3 {
            for t in 0..2 {
                for y in 0..2 {
                    for x in 0..2 {
                        let src = ((((t * 2 + y) * 2 + x) * 3) + c) as f32;
                        want.push(src);
                    }
                }
            }
        }
        assert_eq!(flat, want);
    }
}

#[cfg(test)]
mod hunyuan_tests {
    use super::*;
    use cortiq_core::quant::{GROUP_SIZE, Q1T_TILE, dequant_q1t};

    #[test]
    fn hunyuan_dense_config_builds_ntk_alpha_and_late_qk_norm() {
        let cfg: serde_json::Value = serde_json::from_str(
            r#"{"architectures":["HunYuanDenseV1ForCausalLM"],"model_type":"hunyuan_v1_dense",
                "hidden_size":2048,"intermediate_size":6144,"num_hidden_layers":32,
                "num_attention_heads":16,"num_key_value_heads":4,"head_dim":128,
                "attention_head_dim":128,"vocab_size":120818,"rms_norm_eps":1e-5,
                "rope_theta":10000.0,"max_position_embeddings":262144,
                "rope_scaling":{"alpha":1000.0,"beta_fast":32,"beta_slow":1,"factor":1.0,
                                "mscale":1.0,"mscale_all_dim":1.0,"type":"dynamic"},
                "tie_word_embeddings":true,"use_qk_norm":true,"use_cla":false,
                "norm_type":"rms","hidden_act":"silu","bos_token_id":120000,
                "eos_token_id":120020,"pad_token_id":120002}"#,
        )
        .unwrap();
        let arch = build_arch(&cfg).unwrap();
        assert_eq!(arch.arch_name, "hunyuan_v1_dense");
        assert!(arch.qk_norm_after_rope, "HunYuan dense norms after RoPE");
        // 10000 · 1000^(128/126) — the base llama.cpp writes as rope.freq_base.
        assert!(
            (arch.rope_theta - 11_158_840.0).abs() / 11_158_840.0 < 1e-5,
            "rope_theta {}",
            arch.rope_theta
        );
        assert_eq!(arch.max_position_embeddings, 262_144, "no window cap");
        assert_eq!((arch.head_dim, arch.num_kv_heads, arch.num_attention_heads), (128, 4, 16));
        assert!(arch.tie_word_embeddings);
        assert!(arch.moe.is_none());
        assert_eq!(arch.partial_rotary_factor, 1.0);
        assert!(arch.yarn.is_none());
    }

    #[test]
    fn hunyuan_dense_refuses_cross_layer_attention() {
        let cfg: serde_json::Value = serde_json::from_str(
            r#"{"model_type":"hunyuan_v1_dense","hidden_size":64,"intermediate_size":128,
                "num_hidden_layers":2,"num_attention_heads":4,"num_key_value_heads":2,
                "head_dim":16,"vocab_size":100,"use_cla":true}"#,
        )
        .unwrap();
        let err = build_arch(&cfg).unwrap_err().to_string();
        assert!(err.contains("use_cla"), "{err}");
    }

    #[test]
    fn hy_v3_moe_config_builds_sigmoid_biased_router() {
        let cfg: serde_json::Value = serde_json::from_str(
            r#"{"architectures":["HYV3ForCausalLM"],"model_type":"hy_v3",
                "hidden_size":2048,"intermediate_size":6912,"num_hidden_layers":48,
                "num_attention_heads":32,"num_key_value_heads":4,"head_dim":128,
                "vocab_size":120832,"rms_norm_eps":1e-5,"rope_theta":11158840.0,
                "max_position_embeddings":262144,"tie_word_embeddings":false,
                "num_experts":128,"num_experts_per_tok":8,"moe_intermediate_size":768,
                "expert_hidden_dim":768,"num_shared_experts":1,"first_k_dense_replace":1,
                "moe_router_enable_expert_bias":true,"moe_router_use_sigmoid":true,
                "route_norm":true,"router_scaling_factor":2.826,"qk_norm":true,
                "hidden_act":"silu","bos_token_id":120000,"eos_token_id":120025,
                "eod_token_id":120026,"pad_token_id":120002}"#,
        )
        .unwrap();
        let arch = build_arch(&cfg).unwrap();
        assert_eq!(arch.arch_name, "hy_v3");
        assert!(!arch.qk_norm_after_rope, "hy_v3 keeps the Qwen3 order");
        assert_eq!(arch.rope_theta, 11_158_840.0);
        assert_eq!(arch.intermediate_size, 6912, "layer-0 dense FFN width");
        let moe = arch.moe.as_ref().expect("num_experts → MoE");
        assert_eq!((moe.num_experts, moe.top_k, moe.moe_intermediate_size), (128, 8, 768));
        assert!(moe.norm_topk_prob, "route_norm");
        assert!(moe.router_sigmoid, "moe_router_use_sigmoid");
        assert_eq!(moe.shared_expert_intermediate_size, Some(768));
        assert!((moe.routed_scaling_factor.unwrap() - 2.826).abs() < 1e-6);
        assert!(!moe.router_resonance);
    }

    #[test]
    fn hunyuan_tensor_names_land_on_the_canonical_layout() {
        let m = |s: &str| canon_name(s).unwrap();
        assert_eq!(
            m("model.layers.3.self_attn.query_layernorm.weight"),
            "model.layers.3.self_attn.q_norm.weight"
        );
        assert_eq!(
            m("model.layers.3.self_attn.key_layernorm.weight"),
            "model.layers.3.self_attn.k_norm.weight"
        );
        assert_eq!(
            m("model.layers.1.mlp.router.gate.weight"),
            "model.layers.1.mlp.gate.weight"
        );
        assert_eq!(
            m("model.layers.1.mlp.shared_mlp.up_proj.weight"),
            "model.layers.1.mlp.shared_expert.up_proj.weight"
        );
        assert_eq!(m("model.layers.1.mlp.expert_bias"), "model.layers.1.mlp.expert_bias");
        assert_eq!(
            m("model.layers.1.mlp.experts.17.down_proj.weight"),
            "model.layers.1.mlp.experts.17.down_proj.weight"
        );
        assert!(force_f32("model.layers.1.mlp.expert_bias"));
    }

    #[test]
    fn q1t_exact_reproduces_ternary_weights_bit_for_bit() {
        let (rows, cols) = (3usize, 96usize);
        let mut w = vec![0f32; rows * cols];
        for (g, grp) in w.chunks_mut(GROUP_SIZE).enumerate() {
            let s = f16_to_f32(f32_to_f16(0.013 * (g as f32 + 1.0)));
            for (k, v) in grp.iter_mut().enumerate() {
                *v = match (k * 7 + g) % 3 {
                    0 => 0.0,
                    1 => s,
                    _ => -s,
                };
            }
        }
        let enc = encode_q1t_exact(&w, rows, cols).unwrap();
        assert_eq!(enc.len(), (rows * cols / GROUP_SIZE) * Q1T_TILE + (rows + 1) * 4);
        let mut back = vec![0f32; rows * cols];
        dequant_q1t(&enc, rows, cols, &mut back);
        assert_eq!(back, w);
        // Non-ternary input is refused, never silently rounded.
        let mut bad = w.clone();
        bad[5] = 0.5 * bad.iter().find(|v| **v > 0.0).copied().unwrap();
        assert!(encode_q1t_exact(&bad, rows, cols).is_err());
    }
}

#[cfg(test)]
mod mimo_v2_tests {
    use super::tests::{ENV_LOCK, env_guard};
    use super::*;
    use cortiq_core::format::CmfModel;
    use cortiq_core::quant::dequant_q8_2f;

    /// The text-relevant keys of XiaomiMiMo/MiMo-V2.6-Flash-RL config.json,
    /// verbatim (the vision/audio/processor sub-configs and the 49-entry
    /// ignored_layers list are left out; nothing reads them).
    fn real_config() -> serde_json::Value {
        serde_json::json!({
            "add_full_attention_sink_bias": false,
            "add_swa_attention_sink_bias": true,
            "architectures": ["MiMoV2ForCausalLM"],
            "attention_bias": false,
            "attention_chunk_size": 128,
            "attention_dropout": 0.0,
            "attention_projection_layout": "fused_qkv",
            "attention_value_scale": 0.707,
            "moe_router_dtype": "bfloat16",
            "bos_token_id": null,
            "dtype": "bfloat16",
            "eos_token_id": 151645,
            "head_dim": 192,
            "hidden_act": "silu",
            "hidden_size": 4096,
            "hybrid_block_size": null,
            "hybrid_layer_pattern": [0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1,
                                     1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1,
                                     1, 0, 1, 1, 1, 1, 1, 0],
            "initializer_range": 0.02,
            "intermediate_size": 16384,
            "layernorm_epsilon": 1e-06,
            "max_position_embeddings": 1048576,
            "model_type": "mimo_v2",
            "moe_intermediate_size": 2048,
            "moe_layer_freq": [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
                               1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
                               1, 1, 1, 1],
            "n_group": 1,
            "n_routed_experts": 256,
            "n_shared_experts": null,
            "norm_topk_prob": true,
            "num_attention_heads": 64,
            "num_experts_per_tok": 8,
            "num_hidden_layers": 48,
            "num_key_value_heads": 4,
            "num_nextn_predict_layers": 3,
            "pad_token_id": 151643,
            "partial_rotary_factor": 0.334,
            "quantization_config": {
                "activation_scheme": "dynamic",
                "fmt": "e4m3",
                "ignored_layers": ["model.layers.0.self_attn.o_proj"],
                "mxfp4_block_size": 32,
                "quant_method": "fp8",
                "store_dtype": "mxfp4",
                "weight_block_size": [128, 128]
            },
            "rope_parameters": {
                "partial_rotary_factor": 0.334,
                "rope_theta": 10000000.0,
                "rope_type": "default",
                "type": "default"
            },
            "rope_theta": 10000000.0,
            "routed_scaling_factor": null,
            "scoring_func": "sigmoid",
            "sliding_window": 128,
            "sliding_window_size": 128,
            "swa_head_dim": 192,
            "swa_num_attention_heads": 64,
            "swa_num_key_value_heads": 8,
            "swa_rope_theta": 10000.0,
            "swa_v_head_dim": 128,
            "tie_word_embeddings": false,
            "topk_group": 1,
            "topk_method": "noaux_tc",
            "use_cache": true,
            "v_head_dim": 128,
            "vocab_size": 152576
        })
    }

    #[test]
    fn mimo_v2_real_config_maps_to_exact_cmf_contract() {
        let config = real_config();
        let arch = build_arch(&config).unwrap();
        assert_eq!(arch.arch_name, "mimo_v2");
        assert_eq!(arch.num_layers, 48);
        assert_eq!(arch.hidden_size, 4096);
        assert_eq!(arch.intermediate_size, 16384, "dense layer-0 MLP width");
        assert_eq!(arch.num_attention_heads, 64);
        assert_eq!(arch.num_kv_heads, 4);
        assert_eq!(arch.head_dim, 192);
        assert_eq!(arch.v_head_dim, Some(128));
        assert_eq!(arch.vocab_size, 152576);
        assert!(!arch.tie_word_embeddings);
        assert_eq!(arch.norm_style, NormStyle::Qwen, "MiMo RMSNorm is x̂·w");
        assert_eq!(arch.rms_norm_eps, 1e-6);
        let full: Vec<usize> = arch
            .layer_types
            .iter()
            .enumerate()
            .filter(|(_, t)| matches!(t, LayerType::FullAttention))
            .map(|(i, _)| i)
            .collect();
        assert_eq!(full, vec![0, 5, 11, 17, 23, 29, 35, 41, 47]);
        assert_eq!(
            arch.layer_types
                .iter()
                .filter(|t| matches!(t, LayerType::SlidingAttention))
                .count(),
            39
        );
        let kv = arch.kv_heads_per_layer.as_ref().unwrap();
        assert_eq!(kv.len(), 48);
        for (li, &n) in kv.iter().enumerate() {
            let want = if full.contains(&li) { 4 } else { 8 };
            assert_eq!(n, want, "layer {li}");
        }
        assert_eq!(arch.sliding_window, Some(128));
        assert_eq!(arch.sliding_window_pattern, None);
        assert_eq!(arch.rope_theta, 1e7);
        assert_eq!(arch.rope_local_base_freq, Some(1e4));
        assert_eq!(arch.local_partial_rotary_factor, None);
        // The loader's formula must land on 64 rotary dims.
        assert_eq!(
            (arch.head_dim as f32 * arch.partial_rotary_factor) as usize,
            64
        );
        assert_eq!(arch.partial_rotary_factor, 64.0 / 192.0);
        assert!(arch.yarn.is_none());
        assert!(arch.mtp.is_none(), "the 3-layer MTP head is dropped");
        assert_eq!(arch.max_position_embeddings, 1048576);
        assert_eq!(arch.query_pre_attn_scalar, None, "softmax scale = 192^-0.5");
        let moe = arch.moe.as_ref().unwrap();
        assert_eq!(moe.num_experts, 256);
        assert_eq!(moe.top_k, 8);
        assert_eq!(moe.moe_intermediate_size, 2048);
        assert!(moe.router_sigmoid);
        assert!(moe.norm_topk_prob);
        assert_eq!(moe.shared_expert_intermediate_size, None);
        assert_eq!(moe.routed_scaling_factor, None);
        // Generation stops on all three ids of generation_config.json.
        let gen_cfg = serde_json::json!({
            "bos_token_id": 151643,
            "eos_token_id": [151643, 151645, 151672]
        });
        assert_eq!(eos_ids(&gen_cfg, &config), vec![151643, 151645, 151672]);
        // No --quant: q4tp.
        assert_eq!(default_quant_for_arch(&arch), Quant::Q4TiledP);
        assert_eq!(mimo_v2_value_scale(&config).unwrap(), 0.707);
    }

    #[test]
    fn mimo_v2_refuses_unsupported_config_variants() {
        let cases: Vec<(&str, serde_json::Value, &str)> = vec![
            ("swa_head_dim", serde_json::json!(128), "swa_head_dim"),
            (
                "swa_num_attention_heads",
                serde_json::json!(32),
                "swa_num_attention_heads",
            ),
            ("swa_v_head_dim", serde_json::json!(192), "swa_v_head_dim"),
            (
                "swa_num_key_value_heads",
                serde_json::json!(6),
                "swa_num_key_value_heads",
            ),
            ("n_group", serde_json::json!(8), "n_group"),
            ("topk_group", serde_json::json!(4), "topk_group"),
            ("topk_method", serde_json::json!("greedy"), "topk_method"),
            (
                "attention_projection_layout",
                serde_json::json!("split_qkv"),
                "fused_qkv",
            ),
            (
                "add_full_attention_sink_bias",
                serde_json::json!(true),
                "add_full_attention_sink_bias",
            ),
            ("attention_bias", serde_json::json!(true), "attention_bias"),
            (
                "hybrid_layer_pattern",
                serde_json::json!([0, 1]),
                "hybrid_layer_pattern",
            ),
            (
                "sliding_window_size",
                serde_json::json!(256),
                "sliding_window",
            ),
            (
                "partial_rotary_factor",
                serde_json::json!(0.5),
                "partial_rotary_factor",
            ),
            (
                "rope_parameters",
                serde_json::json!({"rope_type": "yarn", "factor": 4.0, "rope_theta": 1e7,
                                   "partial_rotary_factor": 0.334}),
                "yarn",
            ),
            (
                "quantization_config",
                serde_json::Value::Null,
                "quantization_config",
            ),
        ];
        for (key, value, needle) in cases {
            let mut config = real_config();
            config[key] = value;
            let err = build_arch(&config)
                .err()
                .unwrap_or_else(|| panic!("{key}: accepted"))
                .to_string();
            assert!(err.contains(needle), "{key}: {err}");
        }
        let mut config = real_config();
        config["hybrid_layer_pattern"][3] = serde_json::json!(2);
        assert!(build_arch(&config).is_err());
        let mut config = real_config();
        config["quantization_config"]["weight_block_size"] = serde_json::json!([64, 64]);
        let err = build_arch(&config).unwrap_err().to_string();
        assert!(err.contains("weight_block_size"), "{err}");
        let mut config = real_config();
        config["quantization_config"]["mxfp4_block_size"] = serde_json::json!(16);
        assert!(build_arch(&config).is_err());
    }

    #[test]
    fn mimo_v2_canon_rules_are_arch_gated_and_switchable() {
        let arch = build_arch(&real_config()).unwrap();
        let c = |raw: &str| canon_name_for_arch(&arch, raw, MimoTowers::TextOnly);
        assert_eq!(
            c("model.layers.3.mlp.gate.e_score_correction_bias").as_deref(),
            Some("model.layers.3.mlp.expert_bias")
        );
        assert_eq!(
            c("model.layers.3.self_attn.attention_sink_bias").as_deref(),
            Some("model.layers.3.self_attn.sinks")
        );
        for kept in [
            "model.embed_tokens.weight",
            "lm_head.weight",
            "model.norm.weight",
            "model.layers.3.self_attn.qkv_proj.weight",
            "model.layers.3.self_attn.o_proj.weight",
            "model.layers.3.mlp.gate.weight",
            "model.layers.3.mlp.experts.17.down_proj.weight",
            "model.layers.0.mlp.gate_proj.weight",
            "model.layers.3.input_layernorm.weight",
        ] {
            assert_eq!(c(kept).as_deref(), Some(kept), "{kept}");
        }
        for dropped in [
            "model.mtp.layers.0.self_attn.qkv_proj.weight",
            "model.mtp.layers.2.enorm.weight",
            "visual.blocks.0.attn.qkv.weight",
            "visual.merger.mlp.0.weight",
            "audio_encoder.layers.0.self_attn.q_proj.weight",
            "speech_embeddings.3.weight",
        ] {
            assert_eq!(c(dropped), None, "{dropped}");
        }
        // The multimodal mode keeps the towers (MTP stays dropped).
        let mm = |raw: &str| mimo_v2_canon(raw, MimoTowers::Multimodal);
        assert_eq!(
            mm("visual.merger.mlp.0.weight").as_deref(),
            Some("visual.merger.mlp.0.weight")
        );
        assert_eq!(
            mm("audio_encoder.x.weight").as_deref(),
            Some("audio_encoder.x.weight")
        );
        assert_eq!(
            mm("speech_embeddings.0.weight").as_deref(),
            Some("speech_embeddings.0.weight")
        );
        assert_eq!(mm("model.mtp.layers.0.enorm.weight"), None);
        assert_eq!(
            mm("model.layers.1.mlp.gate.e_score_correction_bias").as_deref(),
            Some("model.layers.1.mlp.expert_bias")
        );
        // Other families keep the generic policy: the MiMo drops and
        // renames do not leak into them.
        let mut other = arch.clone();
        other.arch_name = "qwen3_moe".into();
        assert_eq!(
            canon_name_for_arch(&other, "audio_encoder.x.weight", MimoTowers::Multimodal)
                .as_deref(),
            Some("audio_encoder.x.weight")
        );
        assert_eq!(
            canon_name_for_arch(
                &other,
                "model.layers.3.self_attn.attention_sink_bias",
                MimoTowers::TextOnly
            )
            .as_deref(),
            Some("model.layers.3.self_attn.attention_sink_bias")
        );
        // Dtype rules: bias and sinks exact f32, router f16.
        assert!(force_f32("model.layers.3.self_attn.sinks"));
        assert!(force_f32("model.layers.3.mlp.expert_bias"));
        assert!(force_f16("model.layers.3.mlp.gate.weight"));
    }

    #[test]
    fn mimo_v2_profile_puts_only_experts_on_the_low_bit_plane() {
        let arch = build_arch(&real_config()).unwrap();
        let q = |requested: Quant, name: &str| profile_quant(&arch, requested, name);
        for skeleton in [
            "model.embed_tokens.weight",
            "lm_head.weight",
            "model.layers.0.self_attn.q_proj.weight",
            "model.layers.1.self_attn.k_proj.weight",
            "model.layers.1.self_attn.v_proj.weight",
            "model.layers.1.self_attn.o_proj.weight",
            "model.layers.0.mlp.gate_proj.weight",
            "model.layers.0.mlp.up_proj.weight",
            "model.layers.0.mlp.down_proj.weight",
        ] {
            assert_eq!(q(Quant::Q4TiledP, skeleton), Quant::Q8_2f, "{skeleton}");
            assert_eq!(q(Quant::Q2TiledP, skeleton), Quant::Q8_2f, "{skeleton}");
            assert_eq!(q(Quant::Q8Row, skeleton), Quant::Q8Row, "{skeleton}");
        }
        for e in ["gate_proj", "up_proj", "down_proj"] {
            let n = format!("model.layers.7.mlp.experts.200.{e}.weight");
            assert_eq!(q(Quant::Q4TiledP, &n), Quant::Q4TiledP, "{n}");
        }
        // q2tp: 2-bit gate/up, 4-bit down, 8-bit skeleton.
        assert_eq!(
            q(
                Quant::Q2TiledP,
                "model.layers.7.mlp.experts.200.gate_proj.weight"
            ),
            Quant::Q2TiledP
        );
        assert_eq!(
            q(
                Quant::Q2TiledP,
                "model.layers.7.mlp.experts.200.down_proj.weight"
            ),
            Quant::Q4TiledP
        );
    }

    // ── synthetic fused-qkv fixtures ─────────────────────────────────────

    /// FP8 bytes that are all finite (never the 0x7F/0xFF NaN codes).
    fn fp8_bytes(n: usize, seed: usize) -> Vec<u8> {
        (0..n)
            .map(|i| {
                let mag = ((i * 29 + seed * 13 + 7) % 126) as u8;
                if (i + seed) % 3 == 0 { mag | 0x80 } else { mag }
            })
            .collect()
    }

    /// The expected split, written independently of the converter: walk the
    /// STORED rows in order, find each row's chunk, its section (Q/K/V) and
    /// its scale row on the per-chunk grid, and append it where it belongs.
    fn expected_split(
        packed: &[u8],
        scales: &[f32],
        s_cols: usize,
        rows: usize,
        cols: usize,
        l: MimoQkvLayout,
        vscale: f32,
    ) -> (Vec<f32>, Vec<f32>, Vec<f32>) {
        let t = l.ckpt_tp;
        let q_pc = l.n_heads / t * l.head_dim;
        let k_pc = l.n_kv_heads / t * l.head_dim;
        let v_pc = l.n_kv_heads / t * l.v_head_dim;
        let rpc = q_pc + k_pc + v_pc;
        let blocks_per_chunk = rpc.div_ceil(128);
        let (mut q, mut k, mut v) = (vec![], vec![], vec![]);
        for r in 0..rows {
            let (chunk, within) = (r / rpc, r % rpc);
            let srow = chunk * blocks_per_chunk + within / 128;
            let row: Vec<f32> = (0..cols)
                .map(|c| fp8_e4m3_to_f32(packed[r * cols + c]) * scales[srow * s_cols + c / 128])
                .collect();
            if within < q_pc {
                q.extend(row);
            } else if within < q_pc + k_pc {
                k.extend(row);
            } else {
                v.extend(row.into_iter().map(|x| x * vscale));
            }
        }
        (q, k, v)
    }

    fn check_split(l: MimoQkvLayout, cols: usize) {
        let t = l.ckpt_tp;
        let rpc = (l.n_heads / t + l.n_kv_heads / t) * l.head_dim + l.n_kv_heads / t * l.v_head_dim;
        let rows = t * rpc;
        let s_rows = t * rpc.div_ceil(128);
        let s_cols = cols.div_ceil(128);
        let packed = fp8_bytes(rows * cols, rows);
        // Powers of two, different on neighbouring scale cells: a wrong
        // grid changes values, it cannot hide in rounding.
        let scales: Vec<f32> = (0..s_rows * s_cols)
            .map(|i| ((i % 9) as f32 - 4.0).exp2())
            .collect();
        let (q, k, v) = mimo_v2_split_qkv(
            &packed,
            &scales,
            &[s_rows, s_cols],
            rows,
            cols,
            l,
            128,
            0.707,
        )
        .unwrap();
        let (eq, ek, ev) = expected_split(&packed, &scales, s_cols, rows, cols, l, 0.707);
        assert_eq!(q.len(), l.n_heads * l.head_dim * cols);
        assert_eq!(k.len(), l.n_kv_heads * l.head_dim * cols);
        assert_eq!(v.len(), l.n_kv_heads * l.v_head_dim * cols);
        let bits = |x: &[f32]| x.iter().map(|v| v.to_bits()).collect::<Vec<_>>();
        assert_eq!(bits(&q), bits(&eq), "q {l:?}");
        assert_eq!(bits(&k), bits(&ek), "k {l:?}");
        assert_eq!(bits(&v), bits(&ev), "v {l:?}");
        // Head identity: query head h sits in chunk h / (n_heads / t), and
        // so does its KV head h / (n_heads / n_kv_heads).
        let qh_per_chunk = l.n_heads / t;
        for h in [0, l.n_heads - 1] {
            let chunk = h / qh_per_chunk;
            let src_row = chunk * rpc + (h % qh_per_chunk) * l.head_dim;
            let srow = chunk * rpc.div_ceil(128) + (src_row % rpc) / 128;
            let want = fp8_e4m3_to_f32(packed[src_row * cols]) * scales[srow * s_cols];
            assert_eq!(q[h * l.head_dim * cols], want, "q head {h}");
            let kvh = h / (l.n_heads / l.n_kv_heads);
            assert_eq!(
                kvh / (l.n_kv_heads / t),
                chunk,
                "kv head {kvh} of q head {h}"
            );
        }
        // The continuous-grid count is refused whenever it is a DIFFERENT map.
        if rpc % 128 != 0 {
            let cont = rows.div_ceil(128);
            assert_ne!(cont, s_rows);
            let err = mimo_v2_split_qkv(
                &packed,
                &scales[..cont * s_cols],
                &[cont, s_cols],
                rows,
                cols,
                l,
                128,
                0.707,
            )
            .unwrap_err()
            .to_string();
            assert!(err.contains("scale plane"), "{err}");
        }
    }

    #[test]
    fn mimo_v2_qkv_deinterleave_is_bit_exact_with_chunked_scales() {
        // ckpt_tp = 2, rows per chunk 96+48+32 = 176 (not a multiple of
        // 128): 2 scale rows per chunk → 4, where a continuous grid has 3.
        check_split(
            MimoQkvLayout {
                n_heads: 4,
                n_kv_heads: 2,
                head_dim: 48,
                v_head_dim: 32,
                ckpt_tp: 2,
            },
            160,
        );
        // ckpt_tp = 4 with one KV head per chunk: 96+48+32 = 176 again.
        check_split(
            MimoQkvLayout {
                n_heads: 8,
                n_kv_heads: 4,
                head_dim: 48,
                v_head_dim: 32,
                ckpt_tp: 4,
            },
            130,
        );
        // ckpt_tp = 4 with two KV heads per chunk (the SWA shape): 96+96+64
        // = 256 = two whole blocks — both grids coincide.
        check_split(
            MimoQkvLayout {
                n_heads: 8,
                n_kv_heads: 8,
                head_dim: 48,
                v_head_dim: 32,
                ckpt_tp: 4,
            },
            128,
        );
        // The real geometry (one 128-column block): full layer 4×3392 rows
        // with 108 scale rows, SWA layer 4×3712 rows with 116.
        check_split(
            MimoQkvLayout {
                n_heads: 64,
                n_kv_heads: 4,
                head_dim: 192,
                v_head_dim: 128,
                ckpt_tp: 4,
            },
            128,
        );
        check_split(
            MimoQkvLayout {
                n_heads: 64,
                n_kv_heads: 8,
                head_dim: 192,
                v_head_dim: 128,
                ckpt_tp: 4,
            },
            128,
        );
    }

    #[test]
    fn mimo_v2_qkv_refuses_wrong_row_counts() {
        let l = MimoQkvLayout {
            n_heads: 4,
            n_kv_heads: 2,
            head_dim: 48,
            v_head_dim: 32,
            ckpt_tp: 2,
        };
        let (rows, cols) = (352usize, 128usize);
        let packed = fp8_bytes(rows * cols, 1);
        let scales = vec![1.0f32; 4];
        assert!(mimo_v2_split_qkv(&packed, &scales, &[4, 1], rows, cols, l, 128, 1.0).is_ok());
        // One row short of 2×176.
        let short = &packed[..(rows - 1) * cols];
        assert!(mimo_v2_split_qkv(short, &scales, &[4, 1], rows - 1, cols, l, 128, 1.0).is_err());
        // KV heads that do not split over the chunks.
        let odd = MimoQkvLayout { n_kv_heads: 3, ..l };
        assert!(mimo_v2_split_qkv(&packed, &scales, &[4, 1], rows, cols, odd, 128, 1.0).is_err());
        // A scale plane with the wrong column count.
        assert!(mimo_v2_split_qkv(&packed, &[1.0; 8], &[4, 2], rows, cols, l, 128, 1.0).is_err());
    }

    #[test]
    fn mxfp4_decode_is_thread_count_invariant() {
        let (rows, cols) = (37usize, 96usize);
        let packed: Vec<u8> = (0..rows * cols / 2).map(|i| (i * 53 + 5) as u8).collect();
        let scales: Vec<u8> = (0..rows * cols / 32)
            .map(|i| 118 + (i % 17) as u8)
            .collect();
        let _g = env_guard("1", "1");
        let serial = unpack_mxfp4(&packed, &scales, rows, cols / 2).unwrap();
        unsafe { std::env::set_var("CMF_ENCODE_THREADS", "8") };
        let parallel = unpack_mxfp4(&packed, &scales, rows, cols / 2).unwrap();
        assert_eq!(serial, parallel);
    }

    #[test]
    fn convert_only_spec_parses() {
        assert_eq!(
            parse_convert_only(" model.layers.0.* ,model.layers.1.*").unwrap(),
            vec!["model.layers.0.*", "model.layers.1.*"]
        );
        assert!(parse_convert_only("").is_err());
        assert!(parse_convert_only("a,,b").is_err());
    }

    // ── end to end ───────────────────────────────────────────────────────

    type RawTensor = (String, &'static str, Vec<usize>, Vec<u8>);

    fn raw_safetensors(tensors: &[RawTensor]) -> Vec<u8> {
        let mut header = serde_json::Map::new();
        let mut data = Vec::new();
        for (name, dtype, shape, bytes) in tensors {
            let start = data.len();
            data.extend_from_slice(bytes);
            header.insert(
                name.clone(),
                serde_json::json!({"dtype": dtype, "shape": shape, "data_offsets": [start, data.len()]}),
            );
        }
        let hjson = serde_json::to_vec(&serde_json::Value::Object(header)).unwrap();
        let mut out = (hjson.len() as u64).to_le_bytes().to_vec();
        out.extend_from_slice(&hjson);
        out.extend_from_slice(&data);
        out
    }

    /// bf16 bytes of f32 values that are exactly representable in bf16.
    fn bf16_of(vals: &[f32]) -> Vec<u8> {
        vals.iter()
            .flat_map(|v| ((v.to_bits() >> 16) as u16).to_le_bytes())
            .collect()
    }

    fn bf16_vals(n: usize, seed: usize) -> Vec<f32> {
        (0..n)
            .map(|i| {
                let x = ((i * 31 + seed * 7) % 97) as f32 / 97.0 - 0.5;
                f32::from_bits((x * 0.25).to_bits() & 0xFFFF_0000)
            })
            .collect()
    }

    /// Independent MXFP4 reference (E2M1 LUT, E8M0 2^(k−127), low nibble
    /// first).
    fn mxfp4_ref(packed: &[u8], scales: &[u8], rows: usize, cols: usize) -> Vec<f32> {
        const LUT: [f32; 8] = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0];
        let mut out = Vec::with_capacity(rows * cols);
        for r in 0..rows {
            for c in 0..cols {
                let byte = packed[r * cols / 2 + c / 2];
                let nib = if c % 2 == 0 { byte & 0xF } else { byte >> 4 };
                let mag = LUT[(nib & 7) as usize];
                let s = (scales[r * cols / 32 + c / 32] as f32 - 127.0).exp2();
                out.push(if nib & 8 != 0 { -mag } else { mag } * s);
            }
        }
        out
    }

    const H: usize = 64;
    const NH: usize = 4;
    const HD: usize = 48;
    const VD: usize = 32;
    const CKPT_TP: usize = 2;
    const INTER: usize = 128;
    const MOE_I: usize = 32;
    const N_EXP: usize = 4;
    const VOCAB: usize = 32;
    const PATTERN: [u64; 3] = [0, 1, 1];

    fn nkv(li: usize) -> usize {
        if PATTERN[li] == 1 { 4 } else { 2 }
    }

    fn tiny_config() -> serde_json::Value {
        serde_json::json!({
            "model_type": "mimo_v2",
            "architectures": ["MiMoV2ForCausalLM"],
            "attention_projection_layout": "fused_qkv",
            "attention_bias": false,
            "add_full_attention_sink_bias": false,
            "add_swa_attention_sink_bias": true,
            "attention_value_scale": 0.707,
            "hidden_size": H,
            "intermediate_size": INTER,
            "moe_intermediate_size": MOE_I,
            "num_hidden_layers": PATTERN.len(),
            "hybrid_layer_pattern": PATTERN,
            "moe_layer_freq": [0, 1, 1],
            "num_attention_heads": NH,
            "num_key_value_heads": CKPT_TP,
            "swa_num_attention_heads": NH,
            "swa_num_key_value_heads": 4,
            "head_dim": HD, "swa_head_dim": HD,
            "v_head_dim": VD, "swa_v_head_dim": VD,
            "partial_rotary_factor": 0.334,
            "rope_parameters": {"partial_rotary_factor": 0.334, "rope_theta": 10000000.0,
                                "rope_type": "default", "type": "default"},
            "rope_theta": 10000000.0,
            "swa_rope_theta": 10000.0,
            "sliding_window": 8,
            "sliding_window_size": 8,
            "layernorm_epsilon": 1e-6,
            "n_routed_experts": N_EXP,
            "num_experts_per_tok": 2,
            "n_shared_experts": null,
            "n_group": 1,
            "topk_group": 1,
            "topk_method": "noaux_tc",
            "scoring_func": "sigmoid",
            "norm_topk_prob": true,
            "routed_scaling_factor": null,
            "num_nextn_predict_layers": 1,
            "vocab_size": VOCAB,
            "max_position_embeddings": 4096,
            "tie_word_embeddings": false,
            "hidden_act": "silu",
            "bos_token_id": null,
            "eos_token_id": 3,
            "pad_token_id": 0,
            "quantization_config": {"quant_method": "fp8", "fmt": "e4m3",
                                    "weight_block_size": [128, 128], "store_dtype": "mxfp4",
                                    "mxfp4_block_size": 32, "activation_scheme": "dynamic"}
        })
    }

    struct Fixture {
        dir: std::path::PathBuf,
        /// Expected f32 values of every emitted tensor, by canonical name.
        want: HashMap<String, (Vec<usize>, Vec<f32>)>,
    }

    fn write_fixture(tag: &str) -> Fixture {
        write_fixture_towers(tag, false)
    }

    /// The tiny text checkpoint; with `towers`, the placeholder tower
    /// tensors are replaced by a complete tiny tower set
    /// (`mimo_towers::tests`), the tower config keys are merged into
    /// config.json and `audio_tokenizer/` is written. `want` stays the TEXT
    /// tensors only.
    fn write_fixture_towers(tag: &str, towers: bool) -> Fixture {
        let dir = std::env::temp_dir().join(format!("cortiq-mimo-{tag}-{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        let mut skel: Vec<RawTensor> = Vec::new();
        let mut experts: Vec<RawTensor> = Vec::new();
        let mut want: HashMap<String, (Vec<usize>, Vec<f32>)> = HashMap::new();
        let plain = |skel: &mut Vec<RawTensor>,
                     want: &mut HashMap<String, (Vec<usize>, Vec<f32>)>,
                     name: &str,
                     shape: Vec<usize>,
                     seed: usize| {
            let n: usize = shape.iter().product();
            let v = bf16_vals(n, seed);
            skel.push((name.to_string(), "BF16", shape.clone(), bf16_of(&v)));
            want.insert(name.to_string(), (shape, v));
        };
        plain(
            &mut skel,
            &mut want,
            "model.embed_tokens.weight",
            vec![VOCAB, H],
            1,
        );
        plain(&mut skel, &mut want, "lm_head.weight", vec![VOCAB, H], 2);
        plain(&mut skel, &mut want, "model.norm.weight", vec![H], 3);
        for li in 0..PATTERN.len() {
            let p = format!("model.layers.{li}.");
            let kv = nkv(li);
            plain(
                &mut skel,
                &mut want,
                &format!("{p}input_layernorm.weight"),
                vec![H],
                10 + li,
            );
            plain(
                &mut skel,
                &mut want,
                &format!("{p}post_attention_layernorm.weight"),
                vec![H],
                20 + li,
            );
            plain(
                &mut skel,
                &mut want,
                &format!("{p}self_attn.o_proj.weight"),
                vec![H, NH * VD],
                30 + li,
            );
            // Fused qkv, stored TP-chunked with per-chunk scale blocks.
            let l = MimoQkvLayout {
                n_heads: NH,
                n_kv_heads: kv,
                head_dim: HD,
                v_head_dim: VD,
                ckpt_tp: CKPT_TP,
            };
            let rpc = (NH / CKPT_TP + kv / CKPT_TP) * HD + kv / CKPT_TP * VD;
            let rows = CKPT_TP * rpc;
            let s_rows = CKPT_TP * rpc.div_ceil(128);
            let packed = fp8_bytes(rows * H, li);
            let scales: Vec<f32> = (0..s_rows).map(|i| ((i % 5) as f32 - 2.0).exp2()).collect();
            let (q, k, v) = expected_split(&packed, &scales, 1, rows, H, l, 0.707);
            want.insert(format!("{p}self_attn.q_proj.weight"), (vec![NH * HD, H], q));
            want.insert(format!("{p}self_attn.k_proj.weight"), (vec![kv * HD, H], k));
            want.insert(format!("{p}self_attn.v_proj.weight"), (vec![kv * VD, H], v));
            skel.push((
                format!("{p}self_attn.qkv_proj.weight"),
                "F8_E4M3",
                vec![rows, H],
                packed,
            ));
            skel.push((
                format!("{p}self_attn.qkv_proj.weight_scale_inv"),
                "F32",
                vec![s_rows, 1],
                scales.iter().flat_map(|s| s.to_le_bytes()).collect(),
            ));
            if PATTERN[li] == 1 {
                let sinks = bf16_vals(NH, 40 + li);
                skel.push((
                    format!("{p}self_attn.attention_sink_bias"),
                    "BF16",
                    vec![NH],
                    bf16_of(&sinks),
                ));
                want.insert(format!("{p}self_attn.sinks"), (vec![NH], sinks));
            }
            if li == 0 {
                // Dense FP8 MLP with one scale per 128×128 tile.
                for (t, shape, s) in [
                    ("gate_proj", vec![INTER, H], 0.5f32),
                    ("up_proj", vec![INTER, H], 0.25),
                    ("down_proj", vec![H, INTER], 2.0),
                ] {
                    let n = shape[0] * shape[1];
                    let packed = fp8_bytes(n, 50 + shape[0]);
                    let vals: Vec<f32> = packed.iter().map(|&b| fp8_e4m3_to_f32(b) * s).collect();
                    let name = format!("{p}mlp.{t}.weight");
                    skel.push((name.clone(), "F8_E4M3", shape.clone(), packed));
                    skel.push((
                        format!("{name}_scale_inv"),
                        "F32",
                        vec![shape[0].div_ceil(128), shape[1].div_ceil(128)],
                        s.to_le_bytes().to_vec(),
                    ));
                    want.insert(name, (shape, vals));
                }
            } else {
                plain(
                    &mut skel,
                    &mut want,
                    &format!("{p}mlp.gate.weight"),
                    vec![N_EXP, H],
                    60 + li,
                );
                let bias: Vec<f32> = (0..N_EXP).map(|e| 0.001 * e as f32 + li as f32).collect();
                skel.push((
                    format!("{p}mlp.gate.e_score_correction_bias"),
                    "F32",
                    vec![N_EXP],
                    bias.iter().flat_map(|v| v.to_le_bytes()).collect(),
                ));
                want.insert(format!("{p}mlp.expert_bias"), (vec![N_EXP], bias));
                for e in 0..N_EXP {
                    for (t, rows, cols) in [
                        ("gate_proj", MOE_I, H),
                        ("up_proj", MOE_I, H),
                        ("down_proj", H, MOE_I),
                    ] {
                        let packed: Vec<u8> = (0..rows * cols / 2)
                            .map(|i| (i * 37 + e * 11 + li * 5 + rows) as u8)
                            .collect();
                        let scales: Vec<u8> = (0..rows * cols / 32)
                            .map(|i| 120 + ((i + e) % 9) as u8)
                            .collect();
                        let name = format!("{p}mlp.experts.{e}.{t}.weight");
                        want.insert(
                            name.clone(),
                            (vec![rows, cols], mxfp4_ref(&packed, &scales, rows, cols)),
                        );
                        experts.push((name.clone(), "U8", vec![rows, cols / 2], packed));
                        experts.push((
                            format!("{name}_scale"),
                            "U8",
                            vec![rows, cols / 32],
                            scales,
                        ));
                    }
                }
            }
        }
        // Everything a text-only conversion must drop. The MTP qkv has a
        // shape no split accepts, so processing it would fail the run.
        let half = bf16_of(&[0.5; 256]);
        skel.push((
            "model.mtp.layers.0.self_attn.qkv_proj.weight".into(),
            "F8_E4M3",
            vec![10, H],
            vec![0x38; 10 * H],
        ));
        skel.push((
            "model.mtp.layers.0.self_attn.qkv_proj.weight_scale_inv".into(),
            "F32",
            vec![1, 1],
            1f32.to_le_bytes().to_vec(),
        ));
        skel.push((
            "model.mtp.layers.0.enorm.weight".into(),
            "BF16",
            vec![H],
            bf16_of(&[1.0; H]),
        ));
        let mut config = tiny_config();
        if towers {
            let (main, tok, _) = crate::mimo_towers::tests::tower_fixture_tensors();
            skel.extend(main);
            crate::mimo_towers::tests::write_audio_tokenizer(&dir, &tok);
            let (tcfg, _) = crate::mimo_towers::tests::tiny_configs();
            for (k, v) in tcfg.as_object().unwrap() {
                if config.get(k).is_none() {
                    config[k] = v.clone();
                }
            }
        } else {
            skel.push((
                "visual.blocks.0.attn.qkv.weight".into(),
                "BF16",
                vec![8, 32],
                half.clone(),
            ));
            skel.push((
                "audio_encoder.layers.0.fc1.weight".into(),
                "BF16",
                vec![8, 32],
                half.clone(),
            ));
            skel.push((
                "speech_embeddings.0.weight".into(),
                "BF16",
                vec![8, 32],
                half,
            ));
        }

        let shards = [
            ("model_pp0_ep0_shard0.safetensors", skel),
            ("model_pp0_ep1_shard0.safetensors", experts),
        ];
        let mut weight_map = serde_json::Map::new();
        for (file, tensors) in &shards {
            fs::write(dir.join(file), raw_safetensors(tensors)).unwrap();
            for (name, ..) in tensors {
                weight_map.insert(name.clone(), serde_json::json!(file));
            }
        }
        fs::write(
            dir.join("model.safetensors.index.json"),
            serde_json::json!({"weight_map": weight_map}).to_string(),
        )
        .unwrap();
        fs::write(dir.join("config.json"), config.to_string()).unwrap();
        fs::write(
            dir.join("generation_config.json"),
            r#"{"bos_token_id": 0, "eos_token_id": [0, 3, 5]}"#,
        )
        .unwrap();
        fs::write(dir.join("tokenizer.json"), b"{}").unwrap();
        fs::write(dir.join("chat_template.jinja"), "{{ messages }}").unwrap();
        Fixture { dir, want }
    }

    fn clear_convert_env() {
        unsafe {
            std::env::remove_var(LOCAL_READY_DIR_ENV);
            std::env::remove_var(SOURCE_SHARD_PRIORITY_ENV);
            std::env::remove_var(CONSUME_SOURCE_SHARDS_ENV);
            std::env::remove_var(CONVERT_ONLY_ENV);
        }
    }

    #[test]
    fn mimo_v2_tiny_checkpoint_converts_end_to_end() {
        // The tensor-quant overrides are process-global, and the resume
        // tests steer the local-ready env; both are taken under ENV_LOCK.
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        clear_convert_env();
        set_tensor_quant_overrides(&[]).unwrap();
        let fx = write_fixture("e2e");
        let auto_out = fx.dir.join("auto.cmf");
        let f16_out = fx.dir.join("f16.cmf");
        // One pass, two profiles: `auto` (what a bare `cortiq convert`
        // asks for) and an exact f16 file for the value checks.
        run_convert_multi(
            fx.dir.to_str().unwrap(),
            &[
                (AUTO_QUANT.into(), auto_out.to_str().unwrap().into()),
                ("f16".into(), f16_out.to_str().unwrap().into()),
            ],
            None,
            None,
            None,
            false,
            |_| {},
        )
        .unwrap();

        let auto = CmfModel::open(&auto_out).unwrap();
        let exact = CmfModel::open(&f16_out).unwrap();
        for m in [&auto, &exact] {
            assert!(m.verify().is_empty(), "verify: {:?}", m.verify());
            let a = m.arch();
            assert_eq!(a.arch_name, "mimo_v2");
            assert_eq!(a.num_layers, 3);
            assert_eq!(a.num_kv_heads, CKPT_TP);
            assert_eq!(a.kv_heads_per_layer, Some(vec![2, 4, 4]));
            assert_eq!(a.v_head_dim, Some(VD));
            assert_eq!(a.head_dim, HD);
            assert_eq!(
                a.layer_types,
                vec![
                    LayerType::FullAttention,
                    LayerType::SlidingAttention,
                    LayerType::SlidingAttention
                ]
            );
            assert_eq!(a.sliding_window, Some(8));
            assert_eq!(a.rope_local_base_freq, Some(10000.0));
            assert_eq!((a.head_dim as f32 * a.partial_rotary_factor) as usize, 16);
            assert!(a.mtp.is_none());
            let tok = m.header.tokenizer_config.as_ref().unwrap();
            assert_eq!(tok.eos_token_ids, vec![0, 3, 5]);
            assert_eq!(tok.bos_token_id, None);
            assert_eq!(tok.pad_token_id, Some(0));
            assert_eq!(tok.chat_template.as_deref(), Some("{{ messages }}"));
        }
        let prov = |m: &CmfModel| m.header.provenance.as_ref().unwrap()["weight_quant"].clone();
        assert_eq!(prov(&auto), serde_json::json!("q4tp"));
        assert_eq!(prov(&exact), serde_json::json!("f16"));

        // The directory: exactly the canonical tensors — nothing dropped
        // leaks in, and no fused or scale tensor survives.
        let mut want_names: Vec<&str> = fx.want.keys().map(|s| s.as_str()).collect();
        want_names.sort();
        for m in [&auto, &exact] {
            let mut names: Vec<&str> = m.tensors.iter().map(|t| t.name.as_str()).collect();
            names.sort();
            assert_eq!(names, want_names);
        }

        for (name, (shape, vals)) in &fx.want {
            let e = exact.tensor(name).unwrap();
            assert_eq!(&e.shape, shape, "{name}");
            let bytes = exact.tensor_bytes(name).unwrap();
            let is_f32 = name.ends_with(".mlp.expert_bias") || name.ends_with(".self_attn.sinks");
            if is_f32 {
                assert_eq!(e.dtype, TensorDtype::F32, "{name}");
                let got: Vec<f32> = bytes
                    .chunks_exact(4)
                    .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
                    .collect();
                assert_eq!(&got, vals, "{name}");
            } else {
                // Every value path — de-interleave, chunked scales, the
                // 0.707 V fold, MXFP4 nibbles — lands bit-exactly in f16.
                assert_eq!(e.dtype, TensorDtype::F16, "{name}");
                assert_eq!(bytes, encode_f16(vals).as_slice(), "{name}");
            }

            // The default (q4tp) profile.
            let a = auto.tensor(name).unwrap();
            assert_eq!(&a.shape, shape, "{name}");
            let want_dtype = if is_f32 {
                TensorDtype::F32
            } else if shape.len() == 1 || name.ends_with(".mlp.gate.weight") {
                TensorDtype::F16
            } else if name.contains(".mlp.experts.") {
                TensorDtype::Q4TiledP
            } else {
                TensorDtype::Q8_2f
            };
            assert_eq!(a.dtype, want_dtype, "{name}");
            if a.dtype == TensorDtype::Q8_2f {
                let mut back = vec![0f32; vals.len()];
                dequant_q8_2f(
                    auto.tensor_bytes(name).unwrap(),
                    shape[0],
                    shape[1],
                    &mut back,
                );
                let dot: f64 = back.iter().zip(vals).map(|(a, b)| (a * b) as f64).sum();
                let na: f64 = back.iter().map(|a| (a * a) as f64).sum::<f64>().sqrt();
                let nb: f64 = vals.iter().map(|b| (b * b) as f64).sum::<f64>().sqrt();
                assert!(dot / (na * nb) > 0.9999, "{name}: cos {}", dot / (na * nb));
            }
        }
        // Split attention shapes: q 4·48 everywhere, k/v per layer type.
        let shape = |n: &str| auto.tensor(n).unwrap().shape.clone();
        assert_eq!(
            shape("model.layers.0.self_attn.q_proj.weight"),
            vec![NH * HD, H]
        );
        assert_eq!(
            shape("model.layers.0.self_attn.k_proj.weight"),
            vec![2 * HD, H]
        );
        assert_eq!(
            shape("model.layers.1.self_attn.k_proj.weight"),
            vec![4 * HD, H]
        );
        assert_eq!(
            shape("model.layers.0.self_attn.v_proj.weight"),
            vec![2 * VD, H]
        );
        assert_eq!(
            shape("model.layers.2.self_attn.v_proj.weight"),
            vec![4 * VD, H]
        );
        assert!(auto.tensor("model.layers.0.self_attn.sinks").is_none());
        let _ = fs::remove_dir_all(&fx.dir);
    }

    #[test]
    fn mimo_v2_multimodal_single_file_carries_the_towers() {
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        clear_convert_env();
        set_tensor_quant_overrides(&[]).unwrap();
        let fx = write_fixture_towers("mmfull", true);
        let full = fx.dir.join("full.cmf");
        let text = fx.dir.join("text.cmf");
        run_convert_multi_towers(
            fx.dir.to_str().unwrap(),
            &[(AUTO_QUANT.into(), full.to_str().unwrap().into())],
            None,
            None,
            None,
            false,
            MimoTowers::Multimodal,
            |_| {},
        )
        .unwrap();
        // The same checkpoint in text mode: no tower, blob or tokenizer tensor.
        run_convert(
            fx.dir.to_str().unwrap(),
            AUTO_QUANT,
            text.to_str().unwrap(),
            None,
            None,
            None,
            false,
            |_| {},
        )
        .unwrap();
        let text = CmfModel::open(&text).unwrap();
        let mut text_names: Vec<&str> = text.tensors.iter().map(|t| t.name.as_str()).collect();
        text_names.sort();
        let mut want_text: Vec<&str> = fx.want.keys().map(|s| s.as_str()).collect();
        want_text.sort();
        assert_eq!(text_names, want_text);

        let full = std::sync::Arc::new(CmfModel::open(&full).unwrap());
        assert!(full.verify().is_empty(), "{:?}", full.verify());
        assert_eq!(full.arch().arch_name, "mimo_v2");
        let (tcfg, tat) = crate::mimo_towers::tests::tiny_configs();
        let inv = cortiq_engine::mimo_mm::mimo_tower_inventory(&tcfg, &tat).unwrap();
        assert_eq!(full.tensors.len(), fx.want.len() + inv.len() + 2);
        // Text tensors: byte-identical to the text-mode file.
        for name in &want_text {
            let (a, b) = (full.tensor(name).unwrap(), text.tensor(name).unwrap());
            assert_eq!((a.dtype, &a.shape), (b.dtype, &b.shape), "{name}");
            assert_eq!(
                full.tensor_bytes(name).unwrap(),
                text.tensor_bytes(name).unwrap(),
                "{name}"
            );
        }
        // Towers: the tower policy (q4tp matrices, not the text skeleton's
        // q8_2f), and the engine accepts the file as-is.
        let mm = cortiq_engine::mimo_mm::MimoMm::from_model(&full).unwrap();
        assert_eq!(mm.source, cortiq_engine::mimo_mm::MimoMmSource::SingleFile);
        assert_eq!(
            mm.dtype("visual.blocks.0.attn.qkv.weight"),
            Some(TensorDtype::Q4TiledP)
        );
        assert_eq!(
            mm.dtype("audio_tokenizer.encoder.layers.1.fc1.weight"),
            Some(TensorDtype::Q4TiledP)
        );
        assert_eq!(
            mm.dtype("audio_tokenizer.encoder.quantizer.vq.layers.2._codebook.embed"),
            Some(TensorDtype::F32)
        );
        assert_eq!(
            mm.dtype("speech_embeddings.0.weight"),
            Some(TensorDtype::Bf16)
        );
        let prov = &full.header.provenance.as_ref().unwrap()["mimo_mm"];
        assert_eq!(prov["towers"], "multimodal");
        assert_eq!(prov["codec"]["visual"]["matrices"], "q4tp");
        let _ = fs::remove_dir_all(&fx.dir);
    }

    #[test]
    fn convert_only_probe_writes_a_marked_partial_file() {
        let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        clear_convert_env();
        set_tensor_quant_overrides(&[]).unwrap();
        let fx = write_fixture("only");
        let out = fx.dir.join("probe.cmf");
        unsafe {
            std::env::set_var(CONVERT_ONLY_ENV, "model.layers.1.self_attn.*");
        }
        let r = run_convert(
            fx.dir.to_str().unwrap(),
            "f16",
            out.to_str().unwrap(),
            None,
            None,
            None,
            false,
            |_| {},
        );
        unsafe {
            std::env::remove_var(CONVERT_ONLY_ENV);
        }
        r.unwrap();
        let m = CmfModel::open(&out).unwrap();
        let mut names: Vec<&str> = m.tensors.iter().map(|t| t.name.as_str()).collect();
        names.sort();
        assert_eq!(
            names,
            vec![
                "model.layers.1.self_attn.k_proj.weight",
                "model.layers.1.self_attn.o_proj.weight",
                "model.layers.1.self_attn.q_proj.weight",
                "model.layers.1.self_attn.sinks",
                "model.layers.1.self_attn.v_proj.weight",
            ]
        );
        assert_eq!(
            m.header.provenance.as_ref().unwrap()["partial_convert_only"],
            serde_json::json!(["model.layers.1.self_attn.*"])
        );
        let _ = fs::remove_dir_all(&fx.dir);
    }
    /// A release-format `model_mtp.safetensors` with `n` draft layers (SWA
    /// geometry: 4 KV heads) and the canonical values every sidecar tensor
    /// must hold.
    fn write_mtp_fixture(
        tag: &str,
        n: usize,
        extra: Option<RawTensor>,
    ) -> (std::path::PathBuf, HashMap<String, (Vec<usize>, Vec<f32>)>) {
        let dir =
            std::env::temp_dir().join(format!("cortiq-mimo-mtp-{tag}-{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        let mut cfg = tiny_config();
        cfg["num_nextn_predict_layers"] = serde_json::json!(n);
        fs::write(dir.join("config.json"), cfg.to_string()).unwrap();
        let mut raw: Vec<RawTensor> = Vec::new();
        let mut want: HashMap<String, (Vec<usize>, Vec<f32>)> = HashMap::new();
        for k in 0..n {
            let p = format!("model.mtp.layers.{k}.");
            for (i, (src, dst)) in MIMO_MTP_NORMS.iter().enumerate() {
                let v: Vec<f32> = bf16_vals(H, 300 + k * 10 + i)
                    .iter()
                    .map(|x| f32::from_bits((1.0 + x).to_bits() & 0xFFFF_0000))
                    .collect();
                raw.push((format!("{p}{src}.weight"), "BF16", vec![H], bf16_of(&v)));
                want.insert(format!("{p}{dst}.weight"), (vec![H], v));
            }
            let eh = bf16_vals(H * 2 * H, 320 + k);
            raw.push((
                format!("{p}eh_proj.weight"),
                "BF16",
                vec![H, 2 * H],
                bf16_of(&eh),
            ));
            want.insert(format!("{p}eh_proj.weight"), (vec![H, 2 * H], eh));
            let l = MimoQkvLayout {
                n_heads: NH,
                n_kv_heads: 4,
                head_dim: HD,
                v_head_dim: VD,
                ckpt_tp: CKPT_TP,
            };
            let rpc = (NH / CKPT_TP + 4 / CKPT_TP) * HD + 4 / CKPT_TP * VD;
            let rows = CKPT_TP * rpc;
            let s_rows = CKPT_TP * rpc.div_ceil(128);
            let packed = fp8_bytes(rows * H, 330 + k);
            let scales: Vec<f32> = (0..s_rows)
                .map(|i| ((i % 3) as f32 - 1.0).exp2())
                .collect();
            let (q, kk, v) = expected_split(&packed, &scales, 1, rows, H, l, 0.707);
            want.insert(format!("{p}self_attn.q_proj.weight"), (vec![NH * HD, H], q));
            want.insert(format!("{p}self_attn.k_proj.weight"), (vec![4 * HD, H], kk));
            want.insert(format!("{p}self_attn.v_proj.weight"), (vec![4 * VD, H], v));
            raw.push((
                format!("{p}self_attn.qkv_proj.weight"),
                "F8_E4M3",
                vec![rows, H],
                packed,
            ));
            raw.push((
                format!("{p}self_attn.qkv_proj.weight_scale_inv"),
                "F32",
                vec![s_rows, 1],
                scales.iter().flat_map(|s| s.to_le_bytes()).collect(),
            ));
            let o = bf16_vals(H * NH * VD, 340 + k);
            raw.push((
                format!("{p}self_attn.o_proj.weight"),
                "BF16",
                vec![H, NH * VD],
                bf16_of(&o),
            ));
            want.insert(format!("{p}self_attn.o_proj.weight"), (vec![H, NH * VD], o));
            let sinks = bf16_vals(NH, 350 + k);
            raw.push((
                format!("{p}self_attn.attention_sink_bias"),
                "BF16",
                vec![NH],
                bf16_of(&sinks),
            ));
            want.insert(format!("{p}self_attn.sinks"), (vec![NH], sinks));
            for (t, shape, sc) in [
                ("gate_proj", vec![INTER, H], 0.5f32),
                ("up_proj", vec![INTER, H], 0.25),
                ("down_proj", vec![H, INTER], 2.0),
            ] {
                let cnt = shape[0] * shape[1];
                let packed = fp8_bytes(cnt, 360 + k + shape[0]);
                let vals: Vec<f32> = packed.iter().map(|&b| fp8_e4m3_to_f32(b) * sc).collect();
                let name = format!("{p}mlp.{t}.weight");
                raw.push((name.clone(), "F8_E4M3", shape.clone(), packed));
                raw.push((
                    format!("{name}_scale_inv"),
                    "F32",
                    vec![shape[0].div_ceil(128), shape[1].div_ceil(128)],
                    sc.to_le_bytes().to_vec(),
                ));
                want.insert(name, (shape, vals));
            }
        }
        if let Some(t) = extra {
            raw.push(t);
        }
        fs::write(dir.join("model_mtp.safetensors"), raw_safetensors(&raw)).unwrap();
        // Like the release: the index maps the MTP names to their file.
        let mut map = serde_json::Map::new();
        for (name, ..) in &raw {
            map.insert(name.clone(), serde_json::json!("model_mtp.safetensors"));
        }
        fs::write(
            dir.join("model.safetensors.index.json"),
            serde_json::json!({ "weight_map": map }).to_string(),
        )
        .unwrap();
        (dir, want)
    }

    #[test]
    fn mimo_v2_mtp_sidecar_holds_every_draft_tensor() {
        let (dir, want) = write_mtp_fixture("ok", 2, None);
        let main = dir.join("mimo-q4tp.cmf");
        for (quant, exact) in [("q8_2f", false), ("f16", true)] {
            let out = write_mimo_mtp_sidecar(&dir, &main, quant).unwrap();
            assert_eq!(out, dir.join("mimo-q4tp.mtp.cmf"));
            let m = CmfModel::open(&out).unwrap();
            assert!(m.verify().is_empty(), "verify: {:?}", m.verify());
            let a = m.arch();
            assert_eq!(a.arch_name, "mimo_v2");
            assert_eq!(a.mtp.as_ref().map(|c| c.num_layers), Some(2));
            assert_eq!(a.kv_heads_per_layer, Some(vec![2, 4, 4]));
            assert_eq!(
                m.header.provenance.as_ref().unwrap()["kind"],
                serde_json::json!("mimo_mtp_sidecar")
            );
            let mut names: Vec<&str> = m.tensors.iter().map(|t| t.name.as_str()).collect();
            names.sort();
            let mut want_names: Vec<&str> = want.keys().map(|s| s.as_str()).collect();
            want_names.sort();
            assert_eq!(names, want_names, "exactly the canonical draft tensors");
            for (name, (shape, vals)) in &want {
                let e = m.tensor(name).unwrap();
                assert_eq!(&e.shape, shape, "{name}");
                let bytes = m.tensor_bytes(name).unwrap();
                if shape.len() == 1 {
                    assert_eq!(e.dtype, TensorDtype::F32, "{name}");
                    let got: Vec<f32> = bytes
                        .chunks_exact(4)
                        .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
                        .collect();
                    assert_eq!(&got, vals, "{name}");
                    continue;
                }
                let back: Vec<f32> = if exact {
                    assert_eq!(e.dtype, TensorDtype::F16, "{name}");
                    bytes
                        .chunks_exact(2)
                        .map(|b| f16_to_f32(u16::from_le_bytes([b[0], b[1]])))
                        .collect()
                } else {
                    assert_eq!(e.dtype, TensorDtype::Q8_2f, "{name}");
                    let mut back = vec![0f32; vals.len()];
                    dequant_q8_2f(bytes, shape[0], shape[1], &mut back);
                    back
                };
                if exact {
                    let worst = back
                        .iter()
                        .zip(vals)
                        .map(|(a, b)| (a - b).abs() / b.abs().max(1e-3))
                        .fold(0.0f32, f32::max);
                    assert!(worst < 1e-3, "{name}: rel err {worst}");
                } else {
                    let dot: f64 = back.iter().zip(vals).map(|(a, b)| (a * b) as f64).sum();
                    let na: f64 = back.iter().map(|a| (a * a) as f64).sum::<f64>().sqrt();
                    let nb: f64 = vals.iter().map(|b| (b * b) as f64).sum::<f64>().sqrt();
                    assert!(dot / (na * nb) > 0.9999, "{name}: cos {}", dot / (na * nb));
                }
            }
        }
        // A path that already names a sidecar is kept.
        let direct = write_mimo_mtp_sidecar(&dir, &dir.join("x.mtp.cmf"), "q8_2f").unwrap();
        assert_eq!(direct, dir.join("x.mtp.cmf"));
        // A main-file profile is refused: the sidecar is skeleton.
        assert!(write_mimo_mtp_sidecar(&dir, &main, "q4tp").is_err());
        let _ = fs::remove_dir_all(&dir);
    }

    #[test]
    fn mimo_v2_mtp_sidecar_refuses_unknown_and_missing_tensors() {
        let extra: RawTensor = (
            "model.mtp.layers.0.mlp.shared_expert.weight".into(),
            "BF16",
            vec![H],
            bf16_of(&[0.5; H]),
        );
        let (dir, _) = write_mtp_fixture("extra", 1, Some(extra));
        let err = write_mimo_mtp_sidecar(&dir, &dir.join("m.cmf"), "q8_2f").unwrap_err();
        assert!(err.to_string().contains("unexpected tensor"), "{err}");
        let _ = fs::remove_dir_all(&dir);
        // The config promises two layers, the file holds one.
        let (dir, _) = write_mtp_fixture("short", 1, None);
        let mut cfg = tiny_config();
        cfg["num_nextn_predict_layers"] = serde_json::json!(2);
        fs::write(dir.join("config.json"), cfg.to_string()).unwrap();
        let err = write_mimo_mtp_sidecar(&dir, &dir.join("m.cmf"), "q8_2f").unwrap_err();
        assert!(err.to_string().contains("not found"), "{err}");
        let _ = fs::remove_dir_all(&dir);
    }
}

#[cfg(test)]
mod dspark_sidecar_tests {
    use super::*;
    use cortiq_core::format::CmfModel;

    type RawTensor = (String, &'static str, Vec<usize>, Vec<u8>);

    fn raw_safetensors(tensors: &[RawTensor]) -> Vec<u8> {
        let mut header = serde_json::Map::new();
        let mut data = Vec::new();
        for (name, dtype, shape, bytes) in tensors {
            let start = data.len();
            data.extend_from_slice(bytes);
            header.insert(
                name.clone(),
                serde_json::json!({"dtype": dtype, "shape": shape, "data_offsets": [start, data.len()]}),
            );
        }
        let json = serde_json::to_vec(&serde_json::Value::Object(header)).unwrap();
        let mut out = (json.len() as u64).to_le_bytes().to_vec();
        out.extend_from_slice(&json);
        out.extend_from_slice(&data);
        out
    }

    fn bf16(vals: &[f32]) -> Vec<u8> {
        vals.iter()
            .flat_map(|v| ((v.to_bits() >> 16) as u16).to_le_bytes())
            .collect()
    }

    fn tiny_layout() -> DsparkLayout {
        DsparkLayout {
            hidden: 32,
            intermediate: 64,
            layers: 5,
            heads: 2,
            kv_heads: 1,
            head_dim: 16,
            vocab: 64,
            markov_rank: 32,
            block_size: 15,
            mask_token_id: 17,
            target_layers: MINICPM_DSPARK_TARGET_IDS,
            num_target_layers: 40,
            rope_theta: 10_000_000.0,
            rms_norm_eps: 1e-6,
        }
    }

    fn official_config() -> serde_json::Value {
        serde_json::json!({
            "architectures": [MINICPM_DSPARK_SOURCE_ARCH],
            "model_type": "qwen3",
            "hidden_size": 2048,
            "intermediate_size": 6144,
            "num_hidden_layers": 5,
            "num_attention_heads": 8,
            "num_key_value_heads": 2,
            "head_dim": 256,
            "vocab_size": 248144,
            "draft_vocab_size": 248144,
            "markov_rank": 256,
            "block_size": 15,
            "num_target_layers": 40,
            "rms_norm_eps": 1e-6,
            "rope_parameters": {"rope_theta": 10000000.0},
            "enable_confidence_head": true,
            "confidence_head_with_markov": true,
            "dflash_config": {
                "projector_type": "dspark",
                "attention_mode": "gqa",
                "markov_head_type": "vanilla",
                "markov_rank": 256,
                "mask_token_id": 248076,
                "target_layer_ids": MINICPM_DSPARK_TARGET_IDS,
                "enable_confidence_head": true,
                "confidence_head_with_markov": true,
            }
        })
    }

    fn write_tiny_source(dir: &Path, layout: &DsparkLayout, extra: bool) -> SafeTensors {
        let mut raw = Vec::new();
        for (si, spec) in dspark_tensor_specs(layout).iter().enumerate() {
            let n = spec.shape.iter().product::<usize>();
            let values: Vec<f32> = (0..n)
                .map(|i| {
                    let x = ((i * 31 + si * 17) % 67) as f32 / 128.0 - 0.25;
                    f32::from_bits(x.to_bits() & 0xffff_0000)
                })
                .collect();
            raw.push((spec.source.clone(), "BF16", spec.shape.clone(), bf16(&values)));
        }
        if extra {
            raw.push(("unexpected.weight".into(), "BF16", vec![1], bf16(&[0.0])));
        }
        let source = dir.join("model.safetensors");
        fs::write(&source, raw_safetensors(&raw)).unwrap();
        open_safetensors(&source).unwrap()
    }

    #[test]
    fn dspark_contract_is_explicit_and_sidecar_name_never_aliases_mtp() {
        let layout = DsparkLayout::official(&official_config()).unwrap();
        assert_eq!(layout.target_layers, MINICPM_DSPARK_TARGET_IDS);
        assert_eq!(layout.block_size, 15);
        let mut bad = official_config();
        bad["dflash_config"]["target_layer_ids"] = serde_json::json!([4, 12, 20, 28, 36]);
        assert!(DsparkLayout::official(&bad).is_err());

        let base = Path::new("/tmp/minicpm-q4tp.cmf");
        assert_eq!(
            cortiq_core::dspark_sidecar_path(base),
            PathBuf::from("/tmp/minicpm-q4tp.dspark.cmf")
        );
        assert_eq!(
            cortiq_core::dspark_sidecar_path(Path::new("/tmp/minicpm-q4tp.mtp.cmf")),
            PathBuf::from("/tmp/minicpm-q4tp.mtp.dspark.cmf")
        );
        assert_eq!(
            cortiq_core::dspark_sidecar_path(Path::new("/tmp/minicpm-q4tp.dspark.cmf")),
            PathBuf::from("/tmp/minicpm-q4tp.dspark.cmf")
        );
    }

    #[test]
    fn dspark_schema_refuses_unknown_source_tensor() {
        let dir = std::env::temp_dir().join(format!("cortiq-dspark-extra-{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        let layout = tiny_layout();
        let source = write_tiny_source(&dir, &layout, true);
        let err = validate_dspark_source(&source, &dspark_tensor_specs(&layout)).unwrap_err();
        assert!(err.to_string().contains("expected 62") || err.to_string().contains("unexpected"));
        let _ = fs::remove_dir_all(&dir);
    }

    #[test]
    fn dspark_payload_writer_preserves_namespace_metadata_and_codecs() {
        let dir = std::env::temp_dir().join(format!("cortiq-dspark-write-{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        let layout = tiny_layout();
        let source = write_tiny_source(&dir, &layout, false);
        let specs = dspark_tensor_specs(&layout);
        validate_dspark_source(&source, &specs).unwrap();
        let binding = DsparkBaseBinding {
            filename: "minicpm-q4tp.cmf".into(),
            arch_name: MINICPM_DSPARK_BASE_ARCH.into(),
            trunk_hash: "0123456789abcdef".into(),
            dir_hash: "fedcba9876543210".into(),
        };
        for (label, quant) in [
            ("f16", Quant::F16),
            ("q8", Quant::Q8_2f),
            ("q4tp", Quant::Q4TiledP),
        ] {
            let header = dspark_header(&layout, &binding, &dir, quant).unwrap();
            let output = dir.join(format!("{label}.dspark.cmf"));
            write_dspark_payload(&source, &specs, &header, quant, &output).unwrap();
            let model = CmfModel::open(&output).unwrap();
            validate_written_dspark_sidecar(&model, &layout, &binding, quant).unwrap();
            assert!(model.tensor("dspark.fc.weight").is_some());
            assert!(model.tensor("fc.weight").is_none());
            assert!(model.tensor("model.mtp.layers.0.fc.weight").is_none());
            assert_eq!(
                model.tensor("dspark.hidden_norm.weight").unwrap().dtype,
                TensorDtype::F32
            );
            assert_eq!(
                model.tensor("dspark.fc.weight").unwrap().dtype,
                dspark_matrix_dtype(quant)
            );
            assert_eq!(
                model.header.provenance.as_ref().unwrap()["runtime"]["automatic_loading"],
                serde_json::json!(false)
            );
        }
        let _ = fs::remove_dir_all(&dir);
    }
}

/// Native FP4: the `mxfp4` codec, the `fp4` profile and the NVFP4 decoder.
#[cfg(test)]
#[path = "convert_fp4_tests.rs"]
mod fp4_tests;

#[cfg(test)]
mod hub_retry_tests {
    use super::*;

    #[test]
    fn hub_busy_codes_are_retried_long_others_are_not() {
        for code in [429, 500, 502, 503, 504] {
            let e = anyhow::anyhow!("download https://huggingface.co/x: status code {code}");
            assert!(hub_busy(&e), "{code}");
        }
        for code in [401, 403, 404, 416] {
            let e = anyhow::anyhow!("download https://huggingface.co/x: status code {code}");
            assert!(!hub_busy(&e), "{code}");
        }
        assert!(!hub_busy(&anyhow::anyhow!("connection reset by peer")));
        // a plain error still gives up after `attempts`
        let mut calls = 0;
        let r: anyhow::Result<()> = with_retry(2, || {
            calls += 1;
            anyhow::bail!("connection reset")
        });
        assert!(r.is_err());
        assert_eq!(calls, 2);
    }
}