hf2q 0.1.1

//! Inference engine for GGUF models — load, generate, and serve.
//!
//! ADR-008: single backend (mlx-native).  All candle code has been removed.

pub mod api;
#[allow(dead_code)]
pub mod auto_pipeline;
#[allow(dead_code)]
pub mod cache;
pub mod config;
mod deepseek4_cli;
pub mod encoder_worker_singleton;
// forward_mlx removed — gemma4 moved to inference::models::gemma4 (ADR-038 §3.3)
pub mod forward_mlx_shared;
pub mod forward_prefill;
pub mod forward_prefill_batched;
pub mod gpu;
pub mod header;
#[allow(dead_code)]
pub mod kv_persist;
pub mod layer_ctx;
#[allow(dead_code)]
pub mod load_info;
#[allow(dead_code)]
pub mod multi_model;
// ADR-040 Phase A iter-1 scaffolding — MultiSeqKvCache trait + types.
// Production callsite activation gated on Phase A iter-2+ per-model impls.
#[allow(dead_code)]
pub mod multi_seq_kv;
// ADR-040 Phase B iter-1 scaffolding — Scheduler trait + FifoSchedulerAdapter +
// InflightBatchedScheduler signature stub. Production activation gated on
// Phase B iter-3+ + Phase C iter-2 (Engine wiring).
pub mod parity_quality;
#[allow(dead_code)]
#[allow(dead_code)]
pub mod quant_select;
#[allow(dead_code)]
pub mod sampler_pure;
#[allow(dead_code)]
pub mod scheduler;
pub mod spec_decode_cli;

use anyhow::{Context, Result};
use std::path::Path;

use crate::cli;
use crate::debug::INVESTIGATION_ENV;

/// Build a `KernelRegistry` with every shader the embedding forward
/// path needs registered AND compiled. One warmup forward is run
/// against the loaded weights so every `get_pipeline()` call hits the
/// cache thereafter. Returns the warmed registry; caller wraps it in
/// `Arc<Mutex<>>` and stashes in `AppState::embedding_registry` so
/// per-request handlers reuse the cached pipelines instead of paying
/// ~150 ms of shader-compile cost on every `/v1/embeddings` call.
fn build_warmed_embedding_registry(
    em: &api::state::EmbeddingModel,
) -> Result<mlx_native::KernelRegistry> {
    use crate::inference::models::bert::bert_gpu::{
        apply_bert_full_forward_gpu, register_bert_custom_shaders,
    };
    use crate::inference::models::nomic_bert::{
        apply_nomic_bert_full_forward_gpu, register_nomic_bert_kernels,
    };
    use api::state::EmbeddingArch;
    use mlx_native::{DType, KernelRegistry, MlxDevice};

    let arch = em
        .arch
        .as_ref()
        .ok_or_else(|| anyhow::anyhow!("registry warmup: EmbeddingModel has no arch"))?;

    let device =
        MlxDevice::new().map_err(|e| anyhow::anyhow!("registry warmup: MlxDevice::new: {e}"))?;
    let mut registry = KernelRegistry::new();

    // Synthetic warmup input: a single padded-to-32 sequence of [PAD]
    // tokens. The exact ids don't matter — we just need to drive every
    // kernel through one full forward so all pipelines compile. Using
    // pad_id (which is in-vocab) keeps gather happy.
    let seq_len: u32 = 32;
    let pad_id = em.tokenizer.specials().pad;
    let ids: Vec<u32> = vec![pad_id; seq_len as usize];
    let ids_buf = device
        .alloc_buffer((seq_len as usize) * 4, DType::U32, vec![seq_len as usize])
        .map_err(|e| anyhow::anyhow!("registry warmup: alloc ids: {e}"))?;
    // SAFETY: just-allocated u32 buffer; exclusive access.
    unsafe {
        let s: &mut [u32] =
            std::slice::from_raw_parts_mut(ids_buf.contents_ptr() as *mut u32, seq_len as usize);
        s.copy_from_slice(&ids);
    }

    let mut encoder = device
        .command_encoder()
        .map_err(|e| anyhow::anyhow!("registry warmup: command_encoder: {e}"))?;
    let valid_token_count: u32 = 1; // any value ≥ 1 ≤ seq_len works for warmup.

    let _out = match arch {
        EmbeddingArch::Bert { config, weights } => {
            register_bert_custom_shaders(&mut registry);
            apply_bert_full_forward_gpu(
                &mut encoder,
                &mut registry,
                &device,
                &ids_buf,
                None,
                weights,
                config,
                seq_len,
                valid_token_count,
            )?
        }
        EmbeddingArch::NomicBert { config, weights } => {
            register_nomic_bert_kernels(&mut registry);
            apply_nomic_bert_full_forward_gpu(
                &mut encoder,
                &mut registry,
                &device,
                &ids_buf,
                None,
                weights,
                config,
                seq_len,
                valid_token_count,
            )?
        }
    };
    encoder
        .commit_and_wait()
        .map_err(|e| anyhow::anyhow!("registry warmup: commit_and_wait: {e}"))?;

    tracing::info!(
        arch = arch.arch_name(),
        cached_pipelines = registry.cached_count(),
        "Warmed embedding kernel registry"
    );

    Ok(registry)
}

/// Resolve the tokenizer path: explicit flag, or look next to GGUF / in parent dirs.
///
/// Iter-215 Wedge-2: visibility raised to `pub(crate)` so the new
/// `serve::api::engine_qwen35::Qwen35LoadedModel::load` constructor
/// can reuse the same tokenizer-resolution logic `cmd_generate_qwen35`
/// uses (parity with the working CLI chat path).
pub(crate) fn find_tokenizer(
    model_path: &Path,
    explicit: Option<&Path>,
) -> Result<std::path::PathBuf> {
    if let Some(p) = explicit {
        return Ok(p.to_path_buf());
    }
    // Look next to GGUF
    let dir = model_path.parent().unwrap_or(Path::new("."));
    let candidate = dir.join("tokenizer.json");
    if candidate.exists() {
        return Ok(candidate);
    }
    // Look in models/{model_name}/ directory
    let _stem = model_path.file_stem().unwrap_or_default().to_string_lossy();
    // Try common patterns
    for subdir in &["gemma4", "gemma-4"] {
        let candidate = Path::new("models").join(subdir).join("tokenizer.json");
        if candidate.exists() {
            return Ok(candidate);
        }
    }
    // Try to match model name prefix
    let models_dir = Path::new("models");
    if models_dir.is_dir() {
        for entry in std::fs::read_dir(models_dir)? {
            let entry = entry?;
            if entry.path().is_dir() {
                let tok = entry.path().join("tokenizer.json");
                if tok.exists() {
                    return Ok(tok);
                }
            }
        }
    }
    anyhow::bail!(
        "Cannot find tokenizer.json. Tried next to GGUF and in models/. \
         Use --tokenizer to specify the path explicitly."
    )
}

/// Resolve config.json path.
/// Resolve the prompt text from either `--prompt` or `--prompt-file`.
fn resolve_prompt(args: &cli::GenerateArgs) -> Result<String> {
    match (&args.prompt, &args.prompt_file) {
        (Some(text), _) => Ok(text.clone()),
        (None, Some(path)) => {
            let content = std::fs::read_to_string(path)
                .with_context(|| format!("Failed to read prompt file: {}", path.display()))?;
            let trimmed = content.trim().to_string();
            anyhow::ensure!(
                !trimmed.is_empty(),
                "Prompt file is empty: {}",
                path.display()
            );
            Ok(trimmed)
        }
        (None, None) => anyhow::bail!("Either --prompt or --prompt-file must be specified"),
    }
}

/// Detect hardware info for benchmark reporting.
fn detect_hardware_info() -> (String, u64) {
    use crate::core::hardware::HardwareProfiler;

    match HardwareProfiler::detect() {
        Ok(profile) => {
            let mem_gb = profile.total_memory_bytes / (1024 * 1024 * 1024);
            (profile.chip_model, mem_gb)
        }
        Err(_) => ("Unknown".to_string(), 0),
    }
}

/// Number of in-process iterations the `--benchmark` flag runs at each
/// context-length regime.  Matches the original a0952e29 contract and
/// the help text in `src/cli.rs` ("5 consecutive runs, report median
/// and p95 tok/s").
const BENCH_NUM_RUNS: usize = 5;

/// ADR-029 iter-19b — `--benchmark` sweeps three generation-length
/// regimes per run so short-context measurements don't mislead about
/// long-context behavior (see ADR-029 §iter-19 reopening).  Operator
/// can still cap the long regime via `--max-tokens N` (each regime is
/// clamped to `min(regime_target, max_tokens)`).
const BENCH_GEN_LENS: &[usize] = &[200, 1000, 2500];

/// Median of an arbitrarily-ordered f64 slice. Empty → 0.0. Sorts a
/// local copy with NaN-safe ordering (NaNs are treated as greater than
/// everything, which never matters for tok/s values).
fn median_f64(values: &[f64]) -> f64 {
    if values.is_empty() {
        return 0.0;
    }
    let mut sorted: Vec<f64> = values.to_vec();
    sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    let n = sorted.len();
    if n % 2 == 1 {
        sorted[n / 2]
    } else {
        (sorted[n / 2 - 1] + sorted[n / 2]) / 2.0
    }
}

/// P95 of an arbitrarily-ordered f64 slice via nearest-rank. Empty → 0.0.
/// Matches the original a0952e29 helper byte-for-byte (clamped to last
/// index for small samples — at N=5 returns sorted[4]).
fn p95_f64(values: &[f64]) -> f64 {
    if values.is_empty() {
        return 0.0;
    }
    let mut sorted: Vec<f64> = values.to_vec();
    sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    let n = sorted.len();
    let rank = ((0.95 * n as f64).ceil() as usize)
        .saturating_sub(1)
        .min(n - 1);
    sorted[rank]
}

/// Print the unified benchmark summary block. `prefill_tps` is `None`
/// for the Gemma path (which historically reports decode only) and
/// `Some(...)` for the Qwen35 paths. The `Decode tok/s:` line is the
/// MEDIAN — this preserves the regex contract of all downstream parser
/// scripts (`scripts/bench-baseline.sh`, `qwen35_bench.sh`,
/// `p17b_q4k_bench.sh`, `iter44/iter45 aggregators`,
/// `adr-011-phase2-gate.sh`) which match `Decode tok/s: <X>` and would
/// silently break if the line vanished or was renamed.
///
/// `extras` lets the qwen35-spec path tack on its acceptance-rate
/// summary without inventing a new printer.
fn print_benchmark_summary(
    model_path: &Path,
    prompt_tokens: usize,
    generated_per_run: &[usize],
    prefill_tps: Option<&[f64]>,
    decode_tps: &[f64],
    extras: &[(String, String)],
) {
    let (chip, mem_gb) = detect_hardware_info();
    let model_filename = model_path
        .file_name()
        .map(|n| n.to_string_lossy().to_string())
        .unwrap_or_else(|| "unknown".to_string());

    let generated_str = if generated_per_run
        .iter()
        .all(|&g| Some(g) == generated_per_run.first().copied())
    {
        format!("{}", generated_per_run.first().copied().unwrap_or(0))
    } else {
        let parts: Vec<String> = generated_per_run.iter().map(|g| g.to_string()).collect();
        parts.join(",")
    };

    println!();
    println!("=== Benchmark Results ===");
    println!("Hardware: {}, {} GB", chip, mem_gb);
    println!("Model: {}", model_filename);
    println!("Prompt tokens: {}", prompt_tokens);
    println!("Generated tokens: {}", generated_str);
    println!("Runs: {}", decode_tps.len());
    if let Some(pp) = prefill_tps {
        for (i, (&pp_tps, &dec_tps)) in pp.iter().zip(decode_tps.iter()).enumerate() {
            println!(
                "Run {}: prefill {:.1} tok/s, decode {:.1} tok/s",
                i + 1,
                pp_tps,
                dec_tps
            );
        }
    } else {
        for (i, &dec_tps) in decode_tps.iter().enumerate() {
            println!("Run {}: decode {:.1} tok/s", i + 1, dec_tps);
        }
    }
    // Load-bearing lines for downstream parser scripts: emit the
    // MEDIAN under the legacy single-value labels so existing regexes
    // continue to find a sensible value. Order matters: Prefill first
    // (qwen35 historic order), Decode last (matches gemma).
    if let Some(pp) = prefill_tps {
        println!("Prefill tok/s: {:.1}", median_f64(pp));
    }
    println!("Decode tok/s: {:.1}", median_f64(decode_tps));
    println!("Median: {:.1} tok/s", median_f64(decode_tps));
    println!("P95:    {:.1} tok/s", p95_f64(decode_tps));
    for (label, value) in extras {
        println!("{}: {}", label, value);
    }
}

/// Hardcoded fallback chat template used ONLY when no GGUF-embedded template
/// exists and the user has not passed `--chat-template` / `--chat-template-file`.
///
/// **CLI `generate` path only.** This template uses a literal `{{PROMPT}}`
/// placeholder consumed by `String::replace` (see `render_chat_template`
/// below). It is NOT compatible with the API path's minijinja rendering
/// (which iterates a `messages` array). The API path uses
/// [`FALLBACK_GEMMA4_API_CHAT_TEMPLATE`] instead.
///
/// **iter-219b parity-gate fix (2026-05-01):** sibling fix to iter-217's
/// API-path correction (commit `5a1b999`). The pre-fix version had a
/// `<|turn>system\n<|think|><turn|>\n` system block that activates
/// Gemma 4's thinking-mode — the model emits a full `<|channel>thought\n
/// ...<channel|>` reasoning block before any answer content. That
/// triggered the parity-gate divergence the iter-219b release-check
/// caught: llama.cpp reference says `"The answer to 2 + 2 is **4**."`
/// while hf2q produced `"<|channel>thought\n* Question: \"Hello, what
/// i..."`. Common-prefix bytes = 0; all 6 parity checks failed (Gates
/// C/D/E + F).
///
/// The fix mirrors `FALLBACK_GEMMA4_API_CHAT_TEMPLATE` (line 271+):
/// drop the `<|think|>` system marker and append an empty channel block
/// `<|channel>thought\n<channel|>` after `<|turn>model\n`. The empty
/// block closes the channel before content begins, so the model emits
/// the answer directly (matching llama.cpp's behavior with no thinking
/// hint).
///
/// Probe (built binary, daily-driver Gemma 4 GGUF):
///   pre-fix: hf2q output begins `<|channel>thought\n* Question: ...`
///            (168 bytes; common-prefix=0 vs llama ref).
///   post-fix: hf2q output begins `The answer to 2 + 2 is **4**.`
///            (matches llama ref through 29-byte threshold).
pub(crate) const FALLBACK_GEMMA4_CHAT_TEMPLATE: &str =
    "<bos><|turn>user\n{{PROMPT}}<turn|>\n<|turn>model\n<|channel>thought\n<channel|>";

/// Hardcoded fallback chat template for the **API path** (consumed by
/// `src/serve/api/engine.rs::render_chat_prompt_with_tools` via minijinja).
///
/// Iterates the `messages` array so `{{m.content}}` actually renders the
/// caller's user content. The CLI fallback (`FALLBACK_GEMMA4_CHAT_TEMPLATE`)
/// uses a literal `{{PROMPT}}` placeholder consumed by `String::replace`,
/// which minijinja treats as an undefined variable — multi-thousand-token
/// user prompts collapsed to ~14 tokens of boilerplate when the GGUF
/// shipped no embedded template (observed in ADR-017 Phase A0.2b matrix
/// run, all SwapBackInSameCtx cells; the chat-template miss is the root
/// cause of the flat no_cache_ttft sweep that broke ship-gate ratios).
///
/// The role mapping mirrors `render_chat_prompt_with_tools`'s
/// `assistant→model` remap (`<|turn>model` is the assistant marker for
/// Gemma 4); the legacy CLI template embeds a system + `<|think|>`
/// preamble which is tokenization-equivalent across single-turn rounds.
///
/// **iter-217 fix (2026-04-30):** the assistant prefix now appends an
/// **empty channel block** `<|channel>thought\n<channel|>` after
/// `<|turn>model\n`, mirroring the upstream Gemma 4 chat template's
/// behavior when `enable_thinking=false` (the default; see
/// `vllm/examples/tool_chat_template_gemma4.jinja:326-330`). Without this
/// empty-block prime, the model emits a stray `<channel|>` close marker
/// in its first decoded tokens (training expects the channel to be
/// closed before content begins). The `ReasoningSplitter` requires both
/// open + close in output to extract reasoning, so a lone close marker
/// leaked verbatim into `delta.content`. Reproducer pre-fix:
/// `curl /v1/chat/completions {"messages":[{"role":"user","content":"What is the capital of France? Answer in English."}]}`
/// → `"content":"<channel|>The capital of France is **Paris**."`.
/// Post-fix: `"content":"The capital of France is **Paris**."` (close
/// marker absorbed by the prompt-side empty-block).
pub(crate) const FALLBACK_GEMMA4_API_CHAT_TEMPLATE: &str = concat!(
    "<bos>",
    "{%- for m in messages -%}",
    "<|turn>{{ m.role }}\n{{ m.content }}<turn|>\n",
    "{%- endfor -%}",
    "<|turn>model\n",
    "<|channel>thought\n<channel|>",
);

/// Resolve the chat template per ADR-005 Phase 1 priority order:
///
///   1. CLI `--chat-template STRING`
///   2. CLI `--chat-template-file FILE`
///   3. GGUF `tokenizer.chat_template` metadata
///   4. Hardcoded fallback string (last resort)
/// Detect whether a Jinja chat template supports thinking-mode rendering by
/// rendering it twice with synthetic test input — once with
/// `enable_thinking=true`, once with `enable_thinking=false` — and checking
/// that the enabled render actually opens a reasoning block. A plain byte
/// difference is insufficient: llama.cpp's template matrix includes Qwen3
/// templates where `enable_thinking=false` appends an empty `<think></think>`
/// suppressor, while `enable_thinking=true` leaves the assistant prompt bare.
/// Those templates branch on the variable but should still default to
/// thinking off.
///
/// **This is the canonical signal** per peer audit
/// `/tmp/cfa-thinking-detect/peer-detection-report.md`. Mirrors llama.cpp's
/// `compare_thinking_enabled` at `/opt/llama.cpp/common/chat-diff-analyzer.cpp:319-401`,
/// surfaced via `common_chat_templates_support_enable_thinking` at
/// `/opt/llama.cpp/common/chat.cpp:244-257`. The user-facing decision in
/// llama.cpp lives at `/opt/llama.cpp/tools/server/server-context.cpp:1050`:
/// `enable_thinking = params_base.enable_reasoning != 0 && template_supports_thinking`.
///
/// Returns `false` if either render fails — a malformed template can't be
/// probed; the caller takes the safe default. The synthetic test input is
/// a single-character user prompt; no real conversation context is needed
/// because the thinking-mode branch fires only on `add_generation_prompt`
/// (always true in our render path), independent of message content.
///
/// 2026-05-02 — replaces the prior name-substring heuristic
/// `gguf_name_suggests_thinking_capable` which the user correctly rejected:
/// > "Thinking is not determined by having thinking in the model name.
/// >  Thinking is determined by the model class and what is available
/// >  in the gguf file."
fn template_supports_enable_thinking(template_str: &str) -> bool {
    let render_true = render_jinja_template(template_str, "x", Some(true));
    let render_false = render_jinja_template(template_str, "x", Some(false));
    match (render_true, render_false) {
        (Ok(enabled), Ok(disabled)) => rendered_prompt_opens_thinking(&enabled, &disabled),
        // Either render failed → can't probe → safe default false. The caller
        // (`render_chat_template`) will surface the actual render error on
        // the real render attempt, so we don't lose information here.
        _ => false,
    }
}

fn rendered_prompt_opens_thinking(enabled: &str, disabled: &str) -> bool {
    let e = enabled.trim_end();
    let d = disabled.trim_end();
    if e == d {
        return false;
    }
    // Canonical signal (mirrors llama.cpp's `compare_thinking_enabled` end-state
    // for `reasoning_mode::TAG_BASED`): the template TRULY OPENS thinking only
    // when, under `enable_thinking=true`, the rendered prompt ends with at
    // least one UNCLOSED open-marker that the `enable_thinking=false` render
    // does not leave open.  Two falsifier patterns the prior pure-byte-diff
    // implementation mis-classified:
    //   * Qwen3 0.6B suppressor:
    //       TRUE  → `<|im_start|>assistant\n`  (bare)
    //       FALSE → `<|im_start|>assistant\n<think>\n\n</think>\n\n`
    //     FALSE adds a PAIRED `<think></think>` suppressor; TRUE has no open
    //     at the tail.  Bytewise different, but TRUE does not open thinking.
    //   * Gemma 4 ara-abliterated (this fix's reproducer):
    //       TRUE  → `<bos><|turn>system\n<|think|><turn|>\n…<|turn>model\n`
    //       FALSE → `<bos><|turn>user\n…<|turn>model\n<|channel>thought\n<channel|>`
    //     FALSE adds a paired `<|channel>thought\n<channel|>` suppressor; TRUE
    //     is bare at the model-turn tail.  Default-on under the prior byte-diff
    //     produced argmax drift into `서히-own-` repetition loops on
    //     enumerative prompts (operator's chemistry-Format probe 2026-05-17).
    //
    // Counted markers are the two reasoning-tag conventions documented in
    // llama.cpp's `chat-diff-analyzer.cpp` workaround list (lines 36-100):
    // `<think></think>` and `<reasoning></reasoning>`.  Model-class-specific
    // section markers (e.g. Gemma 4's `<|channel>...<channel|>`) are NOT
    // open-markers — they're suppressor primers in the FALSE branch and
    // self-contained system-message annotations in the TRUE branch, neither
    // of which constitutes "opening thinking" for the model to continue
    // from at decode time.
    fn unclosed_thinking_count(s: &str) -> i32 {
        let opens = s.matches("<think>").count() as i32
            + s.matches("<reasoning>").count() as i32
            + s.matches("<thinking>").count() as i32;
        let closes = s.matches("</think>").count() as i32
            + s.matches("</reasoning>").count() as i32
            + s.matches("</thinking>").count() as i32;
        opens - closes
    }
    unclosed_thinking_count(e) > unclosed_thinking_count(d)
}

/// Resolve the `enable_thinking` value to pass to the chat-template Jinja
/// context. Three-state precedence:
///
/// 1. **Explicit `--enable-thinking`** → `Some(true)`
/// 2. **Explicit `--no-thinking`** → `Some(false)`
/// 3. **Default (neither flag set)** → auto-detect via render-and-diff
///    (`template_supports_enable_thinking`):
///    * Template branches on the variable → `Some(true)`
///    * Template doesn't branch (or no template provided) → `Some(false)`
///
/// `template_str` is the resolved chat-template Jinja source. Pass `None`
/// when there's no Jinja template (e.g. the hardcoded Gemma4 fallback); in
/// that case the default resolves to `Some(false)` since the fallback path
/// doesn't go through Jinja at all.
///
/// Mirrors llama.cpp's `--reasoning auto` decision at
/// `/opt/llama.cpp/tools/server/server-context.cpp:1050`. Clap enforces
/// `--enable-thinking` ⊕ `--no-thinking` mutual exclusion via
/// `conflicts_with`; the (true, true) input is unreachable in practice
/// but the function remains total.
fn resolve_enable_thinking(args: &cli::GenerateArgs, template_str: Option<&str>) -> Option<bool> {
    if args.enable_thinking {
        return Some(true);
    }
    if args.no_thinking {
        return Some(false);
    }
    let auto = template_str
        .map(template_supports_enable_thinking)
        .unwrap_or(false);
    Some(auto)
}

fn render_chat_template(
    gguf: &mlx_native::gguf::GgufFile,
    args: &cli::GenerateArgs,
    tokenizer: Option<&tokenizers::Tokenizer>,
    user_prompt: &str,
) -> Result<String> {
    // Step 1: resolve the chat template SOURCE in priority order.
    // Priorities 1-3 are Jinja templates; priority 4 is the hardcoded
    // string-replace fallback (not Jinja, so it doesn't go through
    // `render_jinja_template` and `enable_thinking` is irrelevant for it).
    let template_str: String = if let Some(tmpl) = args.chat_template.as_deref() {
        tracing::info!("Chat template: using CLI --chat-template override");
        tmpl.to_string()
    } else if let Some(path) = args.chat_template_file.as_deref() {
        tracing::info!(
            "Chat template: loading from --chat-template-file {}",
            path.display()
        );
        std::fs::read_to_string(path)
            .with_context(|| format!("Failed to read --chat-template-file {}", path.display()))?
    } else if let Some(tmpl) = gguf.metadata_string("tokenizer.chat_template") {
        tracing::info!(
            "Chat template: using GGUF metadata tokenizer.chat_template ({} chars)",
            tmpl.len()
        );
        tmpl.to_string()
    } else {
        // Priority 4: hardcoded string-replace fallback. NOT Jinja — no
        // enable_thinking branching is available here.
        tracing::warn!(
            "Chat template: no GGUF metadata tokenizer.chat_template and no \
             CLI override; falling back to hardcoded Gemma4 template"
        );
        return Ok(FALLBACK_GEMMA4_CHAT_TEMPLATE.replace("{{PROMPT}}", user_prompt));
    };

    // Step 2: resolve enable_thinking using the resolved template (so the
    // auto-detect render-and-diff probes the SAME template we're about to
    // render with). This is the canonical signal per
    // /opt/llama.cpp/common/chat-diff-analyzer.cpp:319-401.
    let enable_thinking = resolve_enable_thinking(args, Some(template_str.as_str()));

    // Step 3: render with the resolved template + flag. Do not post-edit the
    // rendered chat prompt here: llama.cpp's chat-template path treats the
    // template output as authoritative, and Qwen equivalence depends on the
    // same contract. Decode behavior differences belong in the sampler/stop
    // pipeline, not in prompt surgery.
    let bos_token = resolve_token_text(gguf, tokenizer, "tokenizer.ggml.bos_token_id", "<bos>");
    let eos_token = resolve_token_text(gguf, tokenizer, "tokenizer.ggml.eos_token_id", "<eos>");

    render_jinja_template_with_specials(
        &template_str,
        user_prompt,
        enable_thinking,
        &bos_token,
        &eos_token,
    )
}

/// Render a Jinja2 chat template using minijinja.
///
/// Extends the minijinja environment with Python string methods used by
/// Qwen3.5/3.6 and other HuggingFace chat templates:
///
/// - `str.startswith(prefix)` / `str.endswith(suffix)` — Qwen3.6 multi-step
///   tool-call detection path.
/// - `tojson` filter — common HF template helper.
/// - `raise_exception(msg)` function — Qwen3.6 guard for missing user query
///   (we always supply a user message, so this code path is unreachable; we
///   log a warning and continue rather than aborting template rendering).
fn render_jinja_template(
    template_str: &str,
    user_prompt: &str,
    enable_thinking: Option<bool>,
) -> Result<String> {
    render_jinja_template_with_specials(
        template_str,
        user_prompt,
        enable_thinking,
        "<bos>",
        "<eos>",
    )
}

/// How a chat template's `raise_exception(msg)` call behaves
/// (ADR-005 iter-229 Decision 1).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RaisePolicy {
    /// Warn and continue (render succeeds with UNDEFINED). One-shot
    /// CLI/generate path: it always injects a user message, so raise
    /// sites are unreachable in practice and aborting would be worse
    /// than logging.
    Lenient,
    /// Hard render error carrying the template's own message. API path:
    /// the transcript is client-supplied, so raise sites are reachable
    /// and the message must surface in the 400 body (AC5).
    Strict,
}

/// Shared Jinja environment for chat-template rendering (ADR-005
/// iter-229 Decision 1). BOTH renderers — the one-shot
/// `render_jinja_template_with_specials` below and the API path's
/// `render_chat_prompt_with_tools` (serve/api/engine.rs) — must build
/// their environment here; the two diverging was the root cause of the
/// iter-229 F1/F6 failures (API path had no `tojson`/`raise_exception`,
/// so every `tools`-bearing Qwen 3.6 request 400'd at template line 50).
///
/// Registers:
/// - `tojson` filter — JSON-serializes any value, `"null"` fallback.
/// - `raise_exception(msg)` — behavior per [`RaisePolicy`].
/// - pycompat unknown-method callback — Python str/dict/list method
///   shims (`.get`, `.startswith`, `.split`, …) used by the Gemma 4 and
///   Qwen templates. Purely additive: consulted only after minijinja's
///   native dispatch returns UnknownMethod.
pub(crate) fn build_chat_template_env<'s>(raise_policy: RaisePolicy) -> minijinja::Environment<'s> {
    let mut env = minijinja::Environment::new();

    env.add_filter("tojson", |v: minijinja::Value| {
        serde_json::to_string(&v).unwrap_or_else(|_| "null".to_string())
    });

    match raise_policy {
        RaisePolicy::Lenient => {
            env.add_function("raise_exception", |msg: String| -> minijinja::Value {
                tracing::warn!("chat template raise_exception: {}", msg);
                minijinja::Value::UNDEFINED
            });
        }
        RaisePolicy::Strict => {
            env.add_function(
                "raise_exception",
                |msg: String| -> std::result::Result<minijinja::Value, minijinja::Error> {
                    Err(minijinja::Error::new(
                        minijinja::ErrorKind::InvalidOperation,
                        format!("chat template raise_exception: {msg}"),
                    ))
                },
            );
        }
    }

    env.set_unknown_method_callback(minijinja_contrib::pycompat::unknown_method_callback);
    env
}

fn render_jinja_template_with_specials(
    template_str: &str,
    user_prompt: &str,
    enable_thinking: Option<bool>,
    bos_token: &str,
    eos_token: &str,
) -> Result<String> {
    let mut env = build_chat_template_env(RaisePolicy::Lenient);

    env.add_template("chat", template_str)
        .context("Failed to parse chat template as Jinja2")?;
    let tmpl = env
        .get_template("chat")
        .context("Failed to load parsed chat template")?;
    // 2026-05-02 (per H2 audit, docs/research/decode-test-gap-2026-05-02.md):
    // include `enable_thinking` in the Jinja context. Templates like
    // qwen3-chatml.jinja:147-153 branch on this variable to either open an
    // unfilled `<think>\n` block (default: thinking-mode-on) or emit a
    // pre-closed `<think>\n\n</think>\n\n` (thinking-mode-off, cuing the
    // model to skip directly to the answer). Before this fix the variable
    // was never passed and templates always fired the `else` branch — broken
    // for non-thinking checkpoints loaded under the qwen3-arch dispatch.
    //
    // `Option<bool>` ↦ Jinja: None leaves the variable undefined (legacy
    // behavior preserved); Some(b) sets it to that bool value. minijinja's
    // `context!` macro accepts `Option<bool>` and serializes None as
    // undefined, which is exactly what we want for jinja `is defined` guards.
    let rendered = tmpl
        .render(minijinja::context! {
            messages => vec![
                minijinja::context! { role => "user", content => user_prompt }
            ],
            add_generation_prompt => true,
            bos_token => bos_token,
            eos_token => eos_token,
            enable_thinking => enable_thinking,
        })
        .context("Failed to render chat template")?;
    Ok(rendered)
}

fn resolve_token_id(
    gguf: &mlx_native::gguf::GgufFile,
    tokenizer: &tokenizers::Tokenizer,
    metadata_key: &str,
) -> Option<u32> {
    llama_cpp_special_token_id(gguf, metadata_key)
        .and_then(|id| tokenizer.id_to_token(id).map(|_| id))
}

fn llama_cpp_special_token_id(
    gguf: &mlx_native::gguf::GgufFile,
    metadata_key: &str,
) -> Option<u32> {
    if let Some(id) = gguf.metadata_u32(metadata_key) {
        return Some(id);
    }

    let tokenizer_model = gguf.metadata_string("tokenizer.ggml.model")?;
    llama_cpp_special_token_id_for_model(tokenizer_model, metadata_key)
}

fn llama_cpp_special_token_id_for_model(tokenizer_model: &str, metadata_key: &str) -> Option<u32> {
    match (tokenizer_model, metadata_key) {
        // Mirrors `/opt/llama.cpp/src/llama-vocab.cpp`: for tokenizer model
        // `gpt2`, llama.cpp initializes both BOS and EOS to token id 11 before
        // applying GGUF metadata overrides.
        ("gpt2", "tokenizer.ggml.bos_token_id") | ("gpt2", "tokenizer.ggml.eos_token_id") => {
            Some(11)
        }
        _ => None,
    }
}

fn resolve_token_text(
    gguf: &mlx_native::gguf::GgufFile,
    tokenizer: Option<&tokenizers::Tokenizer>,
    metadata_key: &str,
    template_literal_when_unavailable: &str,
) -> String {
    let Some(tokenizer) = tokenizer else {
        return template_literal_when_unavailable.to_string();
    };
    resolve_token_id(gguf, tokenizer, metadata_key)
        .and_then(|id| tokenizer.id_to_token(id))
        .unwrap_or_else(|| template_literal_when_unavailable.to_string())
}

/// Tokenize a rendered prompt with llama.cpp `common_tokenize(...,
/// add_special=true, parse_special=true)` semantics.
///
/// Thin shim around [`crate::core::tokenizer_adapter::tokenize_with_bos_eos_from_gguf`]
/// — the helper lives in `core/` so non-`serve/` callers (notably
/// `inference/spec_decode/eagle3_orchestrator`) can reuse it without
/// pulling `serve` as a dependency. Retained as a private name in
/// this module to minimize churn at the existing call sites.
fn tokenize_rendered_prompt_llama_style(
    gguf: &mlx_native::gguf::GgufFile,
    tokenizer: &tokenizers::Tokenizer,
    prompt_text: &str,
) -> Result<Vec<u32>> {
    crate::core::tokenizer_adapter::tokenize_with_bos_eos_from_gguf(gguf, tokenizer, prompt_text)
}

/// Run the `generate` subcommand.
///
/// ADR-008: single backend path — loads directly from GGUF into mlx-native.
/// ADR-013 P11: routes `qwen35` / `qwen35moe` GGUF architectures to the
/// dedicated Qwen3.5 forward path (`cmd_generate_qwen35`) before attempting
/// the Gemma4 path, so the two model families share the same CLI surface.
pub fn cmd_generate(args: cli::GenerateArgs) -> Result<()> {
    let model_path = &args.model;
    anyhow::ensure!(
        model_path.exists(),
        "Model not found: {}",
        model_path.display()
    );

    // ADR-007 Path C F-6.1: --kv-bits flag wins over env var. Set the env
    // var BEFORE the engine accesses INVESTIGATION_ENV (LazyLock) so the
    // `tq_codebook_bits` field is parsed from this value. Must run before
    // any other module reads the env, hence early in cmd_generate.
    if let Some(bits) = &args.kv_bits {
        // clap's PossibleValuesParser already validated bits ∈ {"4","5","6","8"}.
        std::env::set_var("HF2Q_TQ_CODEBOOK_BITS", bits);
        tracing::info!("ADR-007 F-6.1: KV codebook bits set via --kv-bits {bits} (overrides HF2Q_TQ_CODEBOOK_BITS env)");
    }

    // --- Architecture detection (fast: metadata-only GGUF open) ---
    {
        let gguf_peek = mlx_native::gguf::GgufFile::open(model_path)
            .map_err(|e| anyhow::anyhow!("GGUF open (arch peek): {e}"))?;
        if let Some(arch) = gguf_peek.metadata_string("general.architecture") {
            use crate::inference::models::qwen35::{
                is_qwen36_gguf, is_qwen3_vl_arch, is_qwen3_vl_moe_arch, ARCH_QWEN35, ARCH_QWEN35MOE,
            };
            // Wedge-4 / iter-227 (2026-05-02): originally a runtime
            // actionable-error dispatch shim that bailed on Qwen3-VL
            // arches because the LM forward path wasn't wired yet.
            //
            // **iter-228a (2026-05-02)**: the SERVE-side load path
            // (`LoadedModel::load`) now routes dense Qwen3-VL through
            // `Qwen3VlTextLoadedModel::load` and short-circuits chat
            // completion to a `qwen3vl_text_forward_pending` 501 (the
            // iter-215 → Wedge-3 split for Qwen3.5/3.6 in miniature).
            //
            // The CLI `hf2q generate` path stays bailed at this site:
            // CLI generate has no chat-completion 501 fallback (it
            // would have to actually decode tokens), so its blocker is
            // iter-228b's LM forward, not iter-228a's load surface.
            // Operators who want to validate the iter-228a load
            // surface should run `hf2q serve --model <Qwen3-VL.gguf>`
            // and confirm `/readyz=200` / `/v1/models` reports the
            // model.
            if is_qwen3_vl_arch(arch) {
                let variant_label = if is_qwen3_vl_moe_arch(arch) {
                    "MoE"
                } else {
                    "dense"
                };
                anyhow::bail!(
                    "Qwen3-VL ({variant_label}, general.architecture = {arch:?}) GGUFs are recognized \
                     but the CLI `hf2q generate` path requires a working LM forward (iter-228b \
                     scope). iter-228a (this commit) lands the SERVE-side load path: \
                     `hf2q serve --model <gguf>` will load this GGUF and surface it on \
                     `/v1/models`, but chat completion still returns HTTP 501 with the \
                     `qwen3vl_text_forward_pending` sentinel until iter-228b wires the \
                     dense transformer forward. For text-only chat today, use a \
                     Qwen3.5 / Qwen3.6 GGUF (full chat path). Model: {}",
                    model_path.display(),
                );
            }
            if arch == ARCH_QWEN35 || arch == ARCH_QWEN35MOE {
                // Wave 5a (ADR-005 Phase 4 ACs 5468/5470 partial): the
                // Qwen3.5 forward path is autoregressive-only (per-token
                // DeltaNet state update; correct at short prefill, slow at
                // long prefill until the W-5b chunk-scan kernel lands).
                // Qwen3.6 GGUFs reuse the same `general.architecture`
                // strings as Qwen3.5, so we must distinguish here via
                // `general.name` and require explicit opt-in to avoid
                // silently shipping a slow long-prefill path.
                if is_qwen36_gguf(&gguf_peek) && !INVESTIGATION_ENV.qwen36_autoreg {
                    anyhow::bail!(
                        "Qwen3.6 GGUF detected (general.name contains 'qwen3.6'), but \
                         autoregressive forward-path support is opt-in. Set \
                         HF2Q_QWEN36_AUTOREG=1 to dispatch through the existing \
                         autoregressive Qwen3.5 path (correct at short prefill; long-prefill \
                         SOTA via chunk-scan kernel deferred to Wave 5b). Model: {}",
                        model_path.display(),
                    );
                }
                tracing::info!("Detected architecture '{}' → routing to Qwen3.5 path", arch);
                return cmd_generate_qwen35(args, gguf_peek);
            }
            if arch == "deepseek4" {
                tracing::info!("Detected architecture 'deepseek4' → routing to native path");
                return deepseek4_cli::cmd_generate(args, gguf_peek);
            }
            if arch != "gemma4" {
                anyhow::bail!(
                    "unsupported GGUF general.architecture={arch:?}; `hf2q generate` supports \
                     gemma4 and the explicitly dispatched Qwen families in this build. Model: {}",
                    model_path.display(),
                );
            }
        } else {
            anyhow::bail!(
                "GGUF is missing required `general.architecture`; refusing to guess Gemma. \
                 Model: {}",
                model_path.display(),
            );
        }
    }

    // ADR-018 C3 — unified load path.
    //
    // cmd_generate previously did manual GGUF-open + weights-load +
    // tokenizer-load (~70 LOC of duplicated logic against
    // `GemmaLoadedModel::load`). Refactor: route through the
    // `*LoadedModel::load` constructor so CLI and SERVE share one load
    // surface. Behaviour-equivalent: the constructor opens the GGUF,
    // parses config, builds GpuContext, resolves tokenizer + chat
    // template + EOS + provenance, and emits the same TTY-aware
    // `\r loading i/n layers` progress line via `header::LoadProgress`
    // (mirrors the previously-inline TTY+verbosity dance at mod.rs:519-531).
    //
    // After the load completes we re-open the GGUF (cheap mmap header
    // parse — microsecond-scale, no tensor reads) for the prompt-time
    // call sites that still need it: `render_chat_template`, BOS-token
    // detection, and `build_load_info`.
    let stderr_is_tty = std::io::IsTerminal::is_terminal(&std::io::stderr());
    let stdout_is_tty = std::io::IsTerminal::is_terminal(&std::io::stdout());

    let load_opts = api::engine::LoadOptions {
        model_path: model_path.clone(),
        tokenizer_path: args.tokenizer.clone(),
        config_path: args.config.clone(),
        // cmd_generate path: no DWQ overlay surface yet (cli::GenerateArgs
        // doesn't carry --dwq-overlay; only ServeArgs does).
        dwq_overlay_path: None,
        // ADR-027 Phase A iter-6b.2: cmd_generate(_qwen35) honors
        // HF2Q_KV_PERSIST so qwen35 cold-process LCP resume works on the
        // CLI. Empty / unset env → None → in-process-only.
        kv_persist_dir: std::env::var("HF2Q_KV_PERSIST")
            .ok()
            .filter(|s| !s.is_empty())
            .map(std::path::PathBuf::from),
    };
    let load_start = std::time::Instant::now();
    let loaded =
        api::engine::GemmaLoadedModel::load(&load_opts).context("GemmaLoadedModel::load")?;
    let load_elapsed = load_start.elapsed();

    // ADR-018 C3: emit the unified 13-line load banner on stdout (dim
    // on TTY) BEFORE any prompt rendering or prefill begins, preserving
    // the legacy ordering `print_header_top → render_prompt → prefill →
    // print_header_prefill → decode`. The banner replaces the old 2-line
    // `print_header_top` shape; its content is sourced from the
    // `LoadInfo` snapshot built by `loaded.build_load_info`.
    //
    // Re-open the GGUF for `build_load_info` (which needs the metadata
    // for arch_str + bpw + bos_token_id + chat_template_source) and for
    // the downstream `render_chat_template` + BOS detection. The GGUF
    // header parse is mmap-backed and the OS page cache is already warm
    // from the load above.
    let gguf = mlx_native::gguf::GgufFile::open(model_path)
        .map_err(|e| anyhow::anyhow!("GGUF re-open (post-load, banner+prompt): {e}"))?;

    let mut info = <api::engine::GemmaLoadedModel as load_info::LoadInfoBuilder>::build_load_info(
        &loaded,
        &gguf,
        load_elapsed,
        None,
        false,
    );

    anyhow::ensure!(
        args.image.is_none() || args.mmproj.is_some(),
        "--image requires --mmproj <path>; pass the projector GGUF that pairs \
         with this base model (e.g. mmproj-gemma4-f16.gguf alongside the main \
         GGUF on the same HF repo)"
    );

    // mmproj-on-generate (iter-2): when `--mmproj <path>` is set, validate
    // the GGUF + load the projector weights onto the Metal device when
    // `--image` is also given (otherwise validate-only is sufficient for
    // banner accuracy). The fully-loaded `LoadedMmproj` is consumed
    // downstream by the vision pipeline (preprocess → ViT forward →
    // soft-token splice) before prefill. Mirrors the SERVE
    // `cmd_serve --mmproj` path (mod.rs:3432) so SERVE + CLI exercise
    // one validation surface.
    let loaded_mmproj: Option<api::state::LoadedMmproj> = if let Some(mmp_path) =
        args.mmproj.as_ref()
    {
        anyhow::ensure!(
            mmp_path.exists(),
            "mmproj not found: {}",
            mmp_path.display()
        );
        let mmp_gguf = mlx_native::gguf::GgufFile::open(mmp_path)
            .map_err(|e| anyhow::anyhow!("mmproj GGUF header parse failed: {e}"))?;
        let mmp_config = crate::inference::vision::mmproj::MmprojConfig::from_gguf(&mmp_gguf)
            .map_err(|e| anyhow::anyhow!("mmproj GGUF config parse failed: {e}"))?;
        let actual_names: Vec<&str> = mmp_gguf.tensor_names();
        crate::inference::vision::mmproj::validate_tensor_set(&mmp_config, &actual_names)
            .map_err(|e| anyhow::anyhow!("mmproj GGUF tensor-set validation: {e}"))?;
        let arch_profile = crate::inference::vision::mmproj::detect_arch_profile_with_projector(
            &mmp_config.projector,
            &actual_names,
        );
        anyhow::ensure!(
            arch_profile.is_supported(),
            "mmproj arch profile is Unknown — neither Gemma 4 SigLIP nor \
                 classic CLIP nor Qwen3-VL SigLIP markers found. hf2q's ViT \
                 forward path cannot dispatch on this file."
        );
        let mmproj_sha256 = mmp_gguf
            .metadata_string("hf2q.mmproj_sha256")
            .map(|s| s.to_string());
        info.vision_projector = Some(load_info::VisionProjector {
            mmproj_path: mmp_path.clone(),
            mmproj_sha256,
        });
        // Load weights on demand: only when --image is set does the
        // generate path need the projector resident on the GPU. This
        // keeps text-only `generate --mmproj` cheap (banner + header
        // parse) while paying full cost (~10s on M5 Max) when the
        // user actually intends to run vision.
        if args.image.is_some() {
            let device = mlx_native::MlxDevice::new()
                .map_err(|e| anyhow::anyhow!("MlxDevice for mmproj load: {e}"))?;
            let weights = crate::inference::vision::mmproj_weights::LoadedMmprojWeights::load(
                &mmp_gguf,
                &mmp_config,
                device,
            )
            .map_err(|e| anyhow::anyhow!("mmproj weight load: {e}"))?;
            let model_id = mmp_path
                .file_stem()
                .map(|s| s.to_string_lossy().into_owned())
                .unwrap_or_else(|| "mmproj".into());
            tracing::info!(
                path = %mmp_path.display(),
                image_size = mmp_config.image_size,
                patch_size = mmp_config.patch_size,
                hidden = mmp_config.hidden_size,
                layers = mmp_config.num_hidden_layers,
                projector = mmp_config.projector.as_str(),
                arch = arch_profile.as_str(),
                tensors_loaded = weights.len(),
                "Loaded mmproj GGUF header + tensor set + weights"
            );
            Some(api::state::LoadedMmproj {
                gguf_path: mmp_path.clone(),
                config: mmp_config,
                arch: arch_profile,
                weights: std::sync::Arc::new(weights),
                model_id,
            })
        } else {
            tracing::info!(
                path = %mmp_path.display(),
                image_size = mmp_config.image_size,
                patch_size = mmp_config.patch_size,
                arch = arch_profile.as_str(),
                "Loaded mmproj GGUF header (no --image; weight load skipped)"
            );
            None
        }
    } else {
        None
    };

    load_info::emit_tracing(&info);
    let mut stdout = std::io::stdout();
    load_info::print_banner(&info, &mut stdout, stdout_is_tty).context("print load banner")?;

    // Partial-move the loaded artifacts into local mutable bindings so
    // the rest of cmd_generate (prefill + decode + profiler) keeps the
    // same `&mut ctx` syntax it had pre-refactor. Rust permits this
    // because `GemmaLoadedModel` has no manual `Drop` impl: each field
    // moves independently. After this point `loaded` is no longer
    // usable as a whole — that's intentional, because none of the
    // remaining cmd_generate body wants the aggregate.
    let mut ctx = loaded.ctx;
    let mut mlx_w = loaded.weights;
    let tokenizer = loaded.tokenizer;

    // Resolve prompt
    let prompt_text_raw = resolve_prompt(&args)?;
    // mmproj-on-generate (iter-2): when --image is set, prepend the
    // family placeholder literal so the rendered chat template emits
    // it inside the user role marker. Mirrors SERVE's
    // rewrite_messages_for_vision_placeholders_family — without the
    // placeholder in user content, the chat template emits zero
    // `<|image|>` markers and expand_image_placeholders fails. Today
    // generate accepts a single --image; if/when --image becomes
    // multi-valued, multiple literals are concatenated in CLI order.
    let prompt_text_with_image_marker = if args.image.is_some() {
        if let Some(mmproj) = loaded_mmproj.as_ref() {
            let family = mmproj.arch.vision_family();
            let placeholder = family.placeholder_token_literal().ok_or_else(|| {
                anyhow::anyhow!(
                    "mmproj arch profile {:?} has no placeholder token literal",
                    mmproj.arch
                )
            })?;
            let (open, close) = family.marker_pair();
            format!("{open}{placeholder}{close}\n{prompt_text_raw}")
        } else {
            prompt_text_raw
        }
    } else {
        prompt_text_raw
    };
    let prompt_text = render_chat_template(
        &gguf,
        &args,
        Some(&tokenizer),
        &prompt_text_with_image_marker,
    )?;

    // ADR-005 1bNEW.0c: dump rendered prompt and exit if requested
    if let Some(dump_path) = INVESTIGATION_ENV.dump_rendered_prompt.as_deref() {
        std::fs::write(dump_path, prompt_text.as_bytes())
            .with_context(|| format!("HF2Q_DUMP_RENDERED_PROMPT: failed to write {dump_path}"))?;
        eprintln!(
            "HF2Q_DUMP_RENDERED_PROMPT: wrote {} bytes to {}",
            prompt_text.len(),
            dump_path
        );
        return Ok(());
    }

    // ADR-038 G4-CFA-5e + ADR-015 iter42: `tokenize_rendered_prompt_llama_style`
    // delegates to `core::tokenizer_adapter::tokenize_with_bos_eos_from_gguf`,
    // which already honors GGUF `tokenizer.ggml.add_bos_token` (and `add_eos_token`).
    // The pre-2026-05-23 defense-in-depth duplicate BOS-prepend at this site
    // is no longer needed.
    let prompt_tokens = tokenize_rendered_prompt_llama_style(&gguf, &tokenizer, &prompt_text)?;
    // mmproj-on-generate (iter-2) — image pipeline.
    //
    // When `--image <path>` is set (and `--mmproj <path>` is set, validated
    // upstream), preprocess the image, run the GPU ViT forward through the
    // shared `inference::vision::pipeline` module, expand the
    // `<|image|>`/family placeholder tokens into per-image runs of
    // placeholder ids, and produce per-image `SoftTokenData` slots
    // carrying the projected vision embeddings as GPU buffers. The
    // expanded prompt + soft tokens drive `forward_prefill_with_soft_tokens`
    // below — at the placeholder positions the per-token embed step
    // reads from the override buffer instead of the embed table, so the
    // model sees the projected vision rows.
    //
    // SERVE's `/v1/chat/completions` runs the same pipeline end-to-end
    // (process_multimodal_content → run_vit_forward →
    // expand_image_placeholders → forward_prefill_with_soft_tokens) so
    // both paths exercise byte-identical vision math.
    let mut soft_tokens_owned: Vec<api::engine::SoftTokenData> = Vec::new();
    let prompt_tokens = if let Some(image_path) = args.image.as_ref() {
        let mmproj = loaded_mmproj
            .as_ref()
            .expect("--mmproj checked above when --image is set");
        let image_input =
            crate::inference::vision::parse_image_url(image_path.to_string_lossy().as_ref())
                .with_context(|| format!("--image: parse {}", image_path.display()))?;
        let bytes = crate::inference::vision::load_image_bytes(&image_input)
            .with_context(|| format!("--image: load {}", image_path.display()))?;
        let preprocessed_input = match mmproj.arch {
            crate::inference::vision::mmproj::ArchProfile::Gemma4Siglip => {
                let cfg = &crate::inference::vision::preprocess::GEMMA4V_PREPROCESS_DEFAULT;
                let pp = crate::inference::vision::preprocess::preprocess_gemma4v(&bytes, cfg)
                    .with_context(|| {
                        format!("--image: gemma4v preprocess {}", image_path.display())
                    })?;
                let source_label = image_path
                    .file_name()
                    .and_then(|s| s.to_str())
                    .unwrap_or("image")
                    .to_string();
                crate::inference::vision::vit_gpu::VisionInput::Gemma4v(
                    crate::inference::vision::vit_gpu::Gemma4vPreprocessedImage {
                        patches: pp.patches,
                        pos_x: pp.pos_x,
                        pos_y: pp.pos_y,
                        n_x: pp.n_x,
                        n_y: pp.n_y,
                        source_label,
                    },
                )
            }
            crate::inference::vision::mmproj::ArchProfile::ClipClassic => {
                let preprocess_cfg = mmproj.config.preprocess_config();
                let pixel_values =
                    crate::inference::vision::preprocess_rgb_chw(&bytes, &preprocess_cfg)
                        .with_context(|| {
                            format!("--image: clip preprocess {}", image_path.display())
                        })?;
                let source_label = image_path
                    .file_name()
                    .and_then(|s| s.to_str())
                    .unwrap_or("image")
                    .to_string();
                crate::inference::vision::vit_gpu::VisionInput::Siglip49(
                    crate::inference::vision::PreprocessedImage {
                        pixel_values,
                        target_size: preprocess_cfg.target_size,
                        pixel_w: None,
                        pixel_h: None,
                        source_label,
                    },
                )
            }
            crate::inference::vision::mmproj::ArchProfile::Qwen3VlSiglip => {
                anyhow::bail!(
                    "--image with Qwen3-VL mmproj on `hf2q generate` is not \
                     supported: the Qwen3-VL text LM forward path lives behind \
                     iter-228b (cmd_generate routes via `qwen3vl_text_forward_pending` \
                     501 today). Use `hf2q serve --mmproj <qwen3vl-mmproj>` for \
                     Qwen3-VL vision; this CLI works for Gemma4 + classic-CLIP."
                );
            }
            crate::inference::vision::mmproj::ArchProfile::Unknown => {
                unreachable!("Unknown arch rejected at mmproj load above")
            }
        };
        let pipeline_out = crate::inference::vision::pipeline::run_vit_forward(
            std::slice::from_ref(&preprocessed_input),
            mmproj,
            mlx_w.hidden_size,
        )
        .context("--image: ViT forward")?;
        let (expanded_tokens, soft_tokens, _image_token_positions) =
            crate::inference::vision::pipeline::expand_image_placeholders(
                &tokenizer,
                &prompt_tokens,
                &pipeline_out.embeddings,
                pipeline_out.family,
                pipeline_out.per_row_floats,
                mlx_w.hidden_size,
            )
            .context("--image: soft-token expansion")?;
        tracing::info!(
            image = %image_path.display(),
            arch = mmproj.arch.as_str(),
            n_image_tokens = pipeline_out.total_image_tokens(),
            forward_ms = pipeline_out.forward_ms,
            prompt_tokens_pre = prompt_tokens.len(),
            prompt_tokens_post = expanded_tokens.len(),
            "vision pipeline complete; prompt expanded with soft tokens"
        );
        soft_tokens_owned = soft_tokens;
        expanded_tokens
    } else {
        prompt_tokens
    };
    tracing::info!("Prompt: {} tokens", prompt_tokens.len());
    if INVESTIGATION_ENV.dump_prompt_tokens {
        eprintln!(
            "HF2Q_DUMP_PROMPT_TOKENS: first10={:?} last10={:?} total={}",
            &prompt_tokens[..prompt_tokens.len().min(10)],
            &prompt_tokens[prompt_tokens.len().saturating_sub(10)..],
            prompt_tokens.len()
        );
        eprintln!("HF2Q_DUMP_PROMPT_TOKENS: full={:?}", prompt_tokens);
    }

    let params = sampler_pure::SamplingParams {
        temperature: args.temperature,
        top_p: args.top_p,
        top_k: args.top_k,
        min_p: args.min_p,
        repetition_penalty: args.repetition_penalty,
        max_tokens: args.max_tokens,
    };

    // --- mlx-native forward pass ---
    use std::io::Write;

    tracing::info!("Running mlx-native forward pass");
    let eos_token_ids: Vec<u32> = vec![1, 106];

    // ADR-030 iter-66 production wire-up — HF2Q_SPEC_DFLASH=1 opt-in
    // path.  When unset (production default), returns Ok(None) and
    // execution falls through to the standard prefill + per-token
    // decode loop below.  When set, loads the z-lab DFlash drafter,
    // runs dispatch_dflash_generate, prints the decoded text and
    // returns Ok(Some(())) — caller exits cmd_generate immediately.
    //
    // Coherence (temp=0): accept-walk consistent with the spec path's
    // OWN batched-prefill verifier argmax (NOT byte-identical to
    // single-token decode — at HEAD 6ad9d04c Option A diverges at
    // ~135 tok via single-token argmax flip; Option C from token 1;
    // see ADR-034 §START HERE Cell C row + G3 gate at line 1501).
    // Performance:
    //   Option A (HF2Q_DFLASH_XLEN_SDPA=1 + HF2Q_FULL_F16_KV=1):
    //     45.7 t/s = 0.40× base 113.2 t/s (1.62× over Option C).
    //   Option C (default, re-prefill from start_pos=0):
    //     28.2 t/s = 0.25× base.
    // Both research-quality; flag is opt-in.
    if let Some(()) = crate::serve::spec_decode_cli::try_dispatch_dflash_spec_decode(
        &mut mlx_w,
        &prompt_tokens,
        args.max_tokens,
        &eos_token_ids,
        args.ignore_eos,
        &tokenizer,
        &mut ctx,
    )? {
        return Ok(());
    }

    // ADR-038 G4-CFA-5e closure (2026-05-23) — HF2Q_SPEC_EAGLE3=1 opt-in path
    // for Gemma 4. Mirrors the qwen35 path at spec_decode_cli.rs:553. Default
    // OFF — falls through to standard prefill+decode when unset. When set,
    // loads the published EAGLE-3 drafter (RedHatAI or thoughtworks),
    // runs Gemma4Eagle3Orchestrator::generate (tree-verify spec decode),
    // returns Ok(Some(())) on completion.
    //
    // Per thoughtworks bench (https://huggingface.co/blog/lujangusface/tw-eagle3-gemma4):
    // 1.72× MT-Bench / 1.48× HumanEval / 1.05-1.14× SWEBench. Coherence vs
    // base autoregressive: depends on drafter training distribution + temp=0
    // greedy argmax sensitivity — measure empirically before relying on
    // upstream numbers.
    if let Some(()) = crate::serve::spec_decode_cli::try_dispatch_gemma4_eagle3_spec_decode(
        &mut mlx_w,
        &prompt_tokens,
        args.max_tokens,
        &eos_token_ids,
        args.ignore_eos,
        &tokenizer,
        &mut ctx,
    )? {
        return Ok(());
    }

    // ADR-030 iter-216 Plan B — HF2Q_SPEC_NGRAM=1 opt-in path.  Uses
    // pure-CPU n-gram proposer from ADR-029 Phase 1 instead of the
    // DFlash drafter.  Workload-specific accelerator: needs ~80%
    // acceptance to beat baseline (per iter-212 verify_prefill cost
    // analysis); theoretically targets quote-heavy / repetitive workloads
    // (translate-this-code, summarize-this-text) per vLLM literature.
    //
    // Empirical at HEAD 6f1d1282 2026-05-22 (Gemma 4 26B-A4B-it Q5_K_M,
    // HF2Q_FULL_F16_KV=1, 128 tok temp=0 greedy):
    //   Fibonacci code-gen:        N-gram 19.6 t/s vs base 113.2 = 0.17× base
    //   "Translate verbatim" prompt: N-gram 16.5 t/s vs base 111.6 = 0.15× base
    // N-gram is NET-NEGATIVE on tested workloads. Even the "repetitive"
    // case doesn't manifest the expected acceleration.
    //
    // Greedy invariant: at temp=0 the N-gram path's accept-walk emits
    // tokens consistent with its OWN batched-prefill verifier's argmax.
    // This is NOT byte-identical to single-token Qwen35/Gemma decode —
    // the batched-prefill verifier kernel (default F16 with blk path,
    // or BF16 via flash_attn_prefill_resume when HF2Q_DFLASH_XLEN_SDPA=1)
    // differs from the F32 flash_attn_vec single-token decode kernel.
    // Argmax flips on close logits compound across rounds, the same
    // precision-flip phenomenon documented for DFlash + MTP K=1 BATCHED.
    // Empirical at HEAD 6f1d1282 on Fibonacci prompt: N-gram emits
    // "Here is the most efficient way to implement an iterative
    // Fibonacci function" while base emits "Here is the most efficient
    // way to implement this using an iterative approach" — diverges
    // from token 1.
    //
    // Default OFF.
    if let Some(()) = crate::serve::spec_decode_cli::try_dispatch_ngram_spec_decode(
        &mut mlx_w,
        &prompt_tokens,
        args.max_tokens,
        &eos_token_ids,
        args.ignore_eos,
        &tokenizer,
        &mut ctx,
    )? {
        return Ok(());
    }

    // Profiling support
    let mut profiler = crate::inference::models::gemma4::ProfileAccumulator::new(2);
    let kernel_profile_mode = INVESTIGATION_ENV.mlx_kernel_profile;

    // Prefill: true batched prefill with dense SDPA (ADR-009 Track 1).
    // Uses dense F32 attention instead of TQ-packed attention during prompt
    // ingestion to eliminate compounding quantization noise.
    // ADR-009 Phase 3A: HF2Q_BATCHED_PREFILL=1 uses the new batched prefill
    // path (matches llama.cpp default).
    //
    // ADR-028 iter-344 status: batched prefill is now the DEFAULT-ON path
    // and is DECOUPLED from the `HF2Q_UNSAFE_EXPERIMENTS` ack (iter-343
    // falsifier-passes: 5/5 short + 1000-tok + 4K long-context coherent).
    // Batched prefill at short-to-medium prompt scale (≤80 tokens) is byte-
    // identical to per-token per iter-65/74 (strict byte-equality of
    // decoded outputs), iter-79 (4.3× TTFT speedup with first decode
    // token id=8409 byte-identical), and protected by the iter-76 lock
    // chain (`scripts/adr010_iter64_iter68_full_lock.sh`). The remaining
    // item is the LONG-SEQUENCE sliding_wrap byte-parity *coherence*
    // deferral (ADR-010 L6 MoE router top-K sensitivity, operator-signed
    // 2026-04-16) — not a runtime error. Opt out to per-token for parity
    // diagnostics via `HF2Q_BATCHED_PREFILL=0`.
    let use_batched = INVESTIGATION_ENV.batched_prefill;

    // ADR-005 1bNEW.0 — `--benchmark` runs `BENCH_NUM_RUNS` in-process
    // iterations and reports median + P95 decode tok/s. The help text in
    // `src/cli.rs` ("5 consecutive runs, report median and p95 tok/s")
    // is the binding contract; the original 5-run loop landed in
    // a0952e29 and was incidentally collapsed to a single run in the
    // 9e23b7d8 ADR-008 candle divorce (47k lines deleted). This branch
    // restores the contract.
    //
    // Each iter calls `forward_prefill[_batched]` with `start_pos=0`,
    // which internally resets `cache.write_pos = 0` and reallocates the
    // per-layer `dense_kvs` buffers (see
    // `src/serve/forward_prefill.rs:386` docstring and
    // `forward_prefill_batched.rs:281` allocator). No external KV
    // reset call is needed.
    //
    // Streaming, profiler, kernel-profile, and `HF2Q_DUMP_COUNTERS` are
    // intentionally suppressed in benchmark mode: it's a perf
    // measurement, not an interactive run. Use a non-`--benchmark`
    // invocation when you want any of those.
    //
    // The summary block emits the **median** decode tok/s under the
    // legacy `Decode tok/s: <X>` label so downstream parser scripts
    // (`scripts/bench-baseline.sh`, `qwen35_bench.sh`,
    // `p17b_q4k_bench.sh`, `iter44/iter45 aggregators`,
    // `adr-011-phase2-gate.sh`) keep matching without code changes.
    if args.benchmark {
        // ADR-029 iter-19b: sweep three context-length regimes (short/med/long)
        // so a single number doesn't hide long-context regressions.  Each
        // regime is `args.max_tokens.min(BENCH_GEN_LENS[r])` so the user
        // can still cap via --max-tokens.  We re-prefill before each
        // regime because forward_prefill_* / forward_decode mutate KV
        // state inside `mlx_w`.
        let mut regime_results: Vec<(usize, Vec<f64>, Vec<usize>)> = Vec::new();
        for &regime_target in BENCH_GEN_LENS.iter() {
            let regime_cap = regime_target.min(args.max_tokens);
            if regime_cap == 0 {
                continue;
            }
            let mut decode_tps_runs: Vec<f64> = Vec::with_capacity(BENCH_NUM_RUNS);
            let mut generated_per_run: Vec<usize> = Vec::with_capacity(BENCH_NUM_RUNS);
            eprintln!(
                "\n=== Bench regime: gen_target={} (cap={}) ===",
                regime_target, regime_cap
            );
            for run_idx in 0..BENCH_NUM_RUNS {
                let last_token = if !soft_tokens_owned.is_empty() {
                    let borrowed: Vec<crate::serve::forward_prefill::SoftTokenInjection<'_>> =
                        soft_tokens_owned
                            .iter()
                            .map(|s| crate::serve::forward_prefill::SoftTokenInjection {
                                range: s.range.clone(),
                                embeddings: &s.embeddings,
                            })
                            .collect();
                    mlx_w.forward_prefill_with_soft_tokens(
                        &prompt_tokens,
                        &borrowed,
                        regime_cap,
                        &mut ctx,
                    )?
                } else if use_batched {
                    mlx_w.forward_prefill_batched(&prompt_tokens, regime_cap, 0, &mut ctx)?
                } else {
                    mlx_w.forward_prefill(&prompt_tokens, regime_cap, &mut ctx)?
                };

                let mut all_tokens = prompt_tokens.to_vec();
                let mut next_token = last_token;
                all_tokens.push(next_token);
                let mut decoded_tokens: Vec<u32> = vec![next_token];

                let decode_start = std::time::Instant::now();
                let mut generated = 1usize;
                let mut p: Option<crate::inference::models::gemma4::TokenProfile> = None;
                for _ in 1..regime_cap {
                    if !args.ignore_eos && eos_token_ids.contains(&next_token) {
                        break;
                    }
                    let pos = all_tokens.len() - 1;
                    next_token = mlx_w.forward_decode(next_token, pos, &mut ctx, &mut p)?;
                    all_tokens.push(next_token);
                    generated += 1;
                    decoded_tokens.push(next_token);
                    if detect_greedy_repetition_loop(&decoded_tokens).is_some() {
                        break;
                    }
                }
                let decode_elapsed = decode_start.elapsed();
                let tps = if decode_elapsed.as_secs_f64() > 0.0 {
                    generated as f64 / decode_elapsed.as_secs_f64()
                } else {
                    0.0
                };
                eprintln!(
                    "  [gen={}] Run {}/{}: {} tokens in {:.2}s ({:.1} tok/s)",
                    regime_target,
                    run_idx + 1,
                    BENCH_NUM_RUNS,
                    generated,
                    decode_elapsed.as_secs_f64(),
                    tps,
                );
                decode_tps_runs.push(tps);
                generated_per_run.push(generated);
            }
            regime_results.push((regime_target, decode_tps_runs, generated_per_run));
        }
        // Multi-regime summary block — print each regime's median/P95 + a
        // single rolled-up table for cross-regime comparison.
        println!();
        println!("=== Benchmark Results (multi-regime) ===");
        let (chip, mem_gb) = detect_hardware_info();
        let model_filename = model_path
            .file_name()
            .map(|n| n.to_string_lossy().to_string())
            .unwrap_or_else(|| "unknown".to_string());
        println!("Hardware: {}, {} GB", chip, mem_gb);
        println!("Model: {}", model_filename);
        println!("Prompt tokens: {}", prompt_tokens.len());
        println!("Runs per regime: {}", BENCH_NUM_RUNS);
        println!();
        println!(
            "{:>10} {:>10} {:>10} {:>10}",
            "regime", "median", "p95", "min/max"
        );
        for (target, tps_runs, _gen_per_run) in &regime_results {
            let med = median_f64(tps_runs);
            let p95 = p95_f64(tps_runs);
            let min = tps_runs.iter().copied().fold(f64::INFINITY, f64::min);
            let max = tps_runs.iter().copied().fold(f64::NEG_INFINITY, f64::max);
            println!(
                "  gen={:>4} {:>10.1} {:>10.1} {:>5.1}/{:.1}",
                target, med, p95, min, max
            );
        }
        // Legacy headline (median of LONGEST regime) for downstream parsers
        // that grep `Decode tok/s:`. Choose the longest regime as the
        // representative production number — short-context wins shouldn't
        // hide long-context regressions in CI.
        if let Some((_, tps_runs, _)) = regime_results.last() {
            println!();
            println!(
                "Decode tok/s: {:.1}  (longest-regime median; full table above)",
                median_f64(tps_runs)
            );
            println!("Median: {:.1} tok/s", median_f64(tps_runs));
            println!("P95:    {:.1} tok/s", p95_f64(tps_runs));
        }
        return Ok(());
    }

    let prefill_start = std::time::Instant::now();
    let last_token = if !soft_tokens_owned.is_empty() {
        // Image-augmented prefill — borrow the owned soft-token buffers
        // for the borrowed-slice contract that
        // forward_prefill_with_soft_tokens expects. The owned `Vec<SoftTokenData>`
        // lives until function exit, so the borrowed slice is valid for
        // the duration of the call. The `_batched` variant doesn't yet
        // accept soft tokens (Phase 2c iter-97 wired the per-token
        // path); CLI image-input thus routes through the per-token
        // prefill regardless of HF2Q_BATCHED_PREFILL.
        let borrowed: Vec<crate::serve::forward_prefill::SoftTokenInjection<'_>> =
            soft_tokens_owned
                .iter()
                .map(|s| crate::serve::forward_prefill::SoftTokenInjection {
                    range: s.range.clone(),
                    embeddings: &s.embeddings,
                })
                .collect();
        mlx_w.forward_prefill_with_soft_tokens(
            &prompt_tokens,
            &borrowed,
            args.max_tokens,
            &mut ctx,
        )?
    } else if use_batched {
        // ADR-028 iter-137: cold prefill always starts at position 0.
        // Future verify_batched (Phase 2 GPU iter-139) passes start_pos > 0.
        mlx_w.forward_prefill_batched(&prompt_tokens, args.max_tokens, 0, &mut ctx)?
    } else {
        mlx_w.forward_prefill(&prompt_tokens, args.max_tokens, &mut ctx)?
    };
    let prefill_elapsed = prefill_start.elapsed();

    // Default-mode header line 3 — prefill stats + blank line framing
    // the generation stream. Stdout, dimmed on TTY.
    let prefill_n = prompt_tokens.len();
    let prefill_ms = prefill_elapsed.as_secs_f64() * 1000.0;
    let prefill_tok_s = if prefill_elapsed.as_secs_f64() > 0.0 {
        prefill_n as f64 / prefill_elapsed.as_secs_f64()
    } else {
        0.0
    };
    header::print_header_prefill(
        &mut stdout,
        &header::HeaderInfoPrefill {
            prefill_n,
            prefill_ms,
            prefill_tok_s,
        },
        stdout_is_tty,
    )
    .context("print header prefill")?;

    // Decode
    //
    // 2026-05-02: mirror cmd_generate_qwen35 cumulative-decode pattern so
    // multi-byte UTF-8 codepoints (emoji, CJK) don't get split at token
    // boundaries → garble like `���`. Also reuse the n-gram repetition
    // guard so Gemma's greedy decode can't run to --max-tokens on a stuck
    // loop. Special-token-string check skipped here — Gemma's marker set
    // (`<end_of_turn>`, `<start_of_turn>`, `<eos>`, `<bos>`) differs from
    // Qwen's; integer-id stop via eos_token_ids.contains() catches `<end_of_turn>`
    // (token id 106) which is the dominant turn-end marker.
    let mut all_tokens = prompt_tokens.to_vec();
    let mut next_token = last_token;
    all_tokens.push(next_token);
    let mut decoded_tokens: Vec<u32> = vec![next_token];
    let mut printed_text = tokenizer.decode(&decoded_tokens, false).unwrap_or_default();
    print!("{}", printed_text);
    std::io::stdout().flush()?;

    // ADR-028 iter-334 — reset mlx-native counters at decode-loop entry so
    // HF2Q_DUMP_COUNTERS=1 reports decode-only dispatch/CB/barrier rates
    // (was previously prefill+decode aggregate, ambiguous per-token attribution).
    if std::env::var("HF2Q_DUMP_COUNTERS").ok().as_deref() == Some("1") {
        mlx_native::reset_counters();
    }
    let decode_start = std::time::Instant::now();
    let mut generated = 1usize;
    let mut kernel_profiles: Vec<crate::inference::models::gemma4::KernelTypeProfile> = Vec::new();
    let kernel_profile_warmup = 2usize;
    let kernel_profile_measure = 3usize;
    for _ in 1..params.max_tokens {
        if !args.ignore_eos && eos_token_ids.contains(&next_token) {
            break;
        }
        let pos = all_tokens.len() - 1;

        let kernel_profile_break = if kernel_profile_mode {
            let (tok, kp) = mlx_w.forward_decode_kernel_profile(next_token, pos, &mut ctx)?;
            next_token = tok;
            if generated > kernel_profile_warmup {
                kernel_profiles.push(kp);
            }
            kernel_profiles.len() >= kernel_profile_measure
        } else {
            let mut p = profiler.start_token();
            // forward_decode populates the on-device logits buffer AND
            // returns the GPU-argmax token id.  If sampling is requested
            // (Phase A of BUG-gemma-cli — wire --temperature /
            // --repetition-penalty / --top-k / --top-p through), pull
            // the logits via logits_view() and override the argmax with
            // a sampler_pure::sample_token result.  Mirrors the qwen35
            // CLI pattern at sample_qwen35_logits_for_generate.  When
            // sampling is NOT needed (greedy temp=0, rep_penalty=1.0,
            // top_k=0, top_p=1.0), keep the GPU-argmax fast-path —
            // no logits download, no host-side compute — preserves
            // the parity gates + benchmarks at full speed.
            let argmax_token = mlx_w.forward_decode(next_token, pos, &mut ctx, &mut p)?;
            next_token = if qwen35_generate_uses_sampling(&args) {
                let mut logits: Vec<f32> = mlx_w
                    .logits_view()
                    .context("Gemma sampling: logits_view")?
                    .to_vec();
                sample_qwen35_logits_for_generate(&mut logits, &args, &decoded_tokens)
            } else {
                argmax_token
            };
            profiler.finish_token(p);
            false
        };
        all_tokens.push(next_token);
        generated += 1;
        decoded_tokens.push(next_token);

        // Cumulative decode + delta print (llama.cpp tok_str_pos pattern).
        let new_full = tokenizer.decode(&decoded_tokens, false).unwrap_or_default();
        if new_full.len() > printed_text.len() && new_full.starts_with(&printed_text) {
            print!("{}", &new_full[printed_text.len()..]);
            std::io::stdout().flush()?;
        }
        printed_text = new_full;

        if kernel_profile_break {
            break;
        }

        // N-gram repetition guard — same shared detector as the qwen35 path.
        // Skipped under --ignore-eos (bench mode wants raw N-token generation).
        // Skipped when sampling is active too — sampling already escapes
        // greedy traps; a trip here under sampling is almost certainly a
        // legitimate repetition pattern that the user opted into.
        if !args.ignore_eos && !qwen35_generate_uses_sampling(&args) {
            // Smart detector: structural punctuation-only cycles (markdown
            // table separators etc.) need more reps before bailing than
            // alphanumeric content loops (real model degeneracy).  Decode
            // the candidate cycle via the tokenizer to classify.
            let detect = detect_greedy_repetition_loop_with_text(&decoded_tokens, |cycle| {
                tokenizer
                    .decode(cycle, /*skip_special_tokens=*/ false)
                    .unwrap_or_default()
            });
            if let Some((ngram, repeats)) = detect {
                tracing::info!(
                    "Gemma decode: greedy n-gram repetition detected (last {} tokens \
                     repeated {} times); stopping. Pass --temperature 0.8 or \
                     --repetition-penalty 1.1 (or use the chat-completion API) to \
                     opt into sampling.",
                    ngram,
                    repeats
                );
                eprintln!(
                    "\n[hf2q] Gemma greedy decode entered a {}-token repetition loop \
                     — stopping.  Pass --temperature 0.8 or --repetition-penalty 1.1 \
                     to escape via sampling.",
                    ngram
                );
                break;
            }
        }
    }
    let decode_elapsed = decode_start.elapsed();
    let tok_per_sec = generated as f64 / decode_elapsed.as_secs_f64();
    let (td, tr) = if stderr_is_tty {
        ("\x1b[2m", "\x1b[0m")
    } else {
        ("", "")
    };
    // Footer line mirrors llama.cpp's headline format:
    //   `[ Prompt: <X> t/s | Generation: <Y> t/s ]`
    // Adds hf2q-specific gen-token-count tail so the legacy
    // "tokens in Xs" data is still in one place for downstream parsers.
    eprintln!(
        "\n\n{td}--- mlx-native: [ Prompt: {:.1} t/s | Generation: {:.1} t/s ]  \
         ({} gen tokens in {:.2}s) ---{tr}",
        prefill_tok_s,
        tok_per_sec,
        generated,
        decode_elapsed.as_secs_f64(),
    );

    // Print profiling summary if enabled
    profiler.print_summary();

    // Print kernel-type profiling report if enabled
    if kernel_profile_mode && !kernel_profiles.is_empty() {
        crate::inference::models::gemma4::MlxModelWeights::print_kernel_profile_report(
            &kernel_profiles,
        );
    }

    // `--benchmark` is handled by the 5-run loop earlier in this
    // function (which `return Ok(())`s before reaching here); the
    // single-run summary block that lived here pre-restoration is
    // unreachable when `args.benchmark` is true. See
    // `if args.benchmark { ... }` above.

    // ADR-005 Gate G: mlx-native dispatch + sync counter emission.
    // Gated by HF2Q_DUMP_COUNTERS=1. Emits totals, per-prompt-token, and
    // per-decode-token rates on stderr so release-check.sh can threshold
    // them. The counters are atomic globals in mlx-native (not per-
    // invocation); for fresh numbers, run hf2q in a fresh process.
    if std::env::var("HF2Q_DUMP_COUNTERS").ok().as_deref() == Some("1") {
        // Counters were reset at decode-loop entry (iter-334) so all values
        // below are decode-only.
        let dispatches = mlx_native::dispatch_count();
        let syncs = mlx_native::sync_count();
        let cmd_bufs = mlx_native::cmd_buf_count();
        let barriers = mlx_native::barrier_count();
        let prompt_n = prompt_tokens.len() as u64;
        let decode_n = generated as u64;
        let dispatches_per_decode_tok = if decode_n > 0 {
            dispatches as f64 / decode_n as f64
        } else {
            0.0
        };
        let syncs_per_decode_tok = if decode_n > 0 {
            syncs as f64 / decode_n as f64
        } else {
            0.0
        };
        let cb_per_decode_tok = if decode_n > 0 {
            cmd_bufs as f64 / decode_n as f64
        } else {
            0.0
        };
        let barriers_per_decode_tok = if decode_n > 0 {
            barriers as f64 / decode_n as f64
        } else {
            0.0
        };
        eprintln!(
            "[MLX_COUNTERS] dispatches={} syncs={} cmd_bufs={} barriers={} \
             prompt_tokens={} decode_tokens={} \
             dispatches/decode_tok={:.2} syncs/decode_tok={:.2} \
             cmd_bufs/decode_tok={:.4} barriers/decode_tok={:.2}",
            dispatches,
            syncs,
            cmd_bufs,
            barriers,
            prompt_n,
            decode_n,
            dispatches_per_decode_tok,
            syncs_per_decode_tok,
            cb_per_decode_tok,
            barriers_per_decode_tok,
        );
        // ADR-028 iter-284: dump per-pipeline dispatch buckets if MLX_DISP_BUCKET=1.
        // Mirrors llama.cpp's LLAMA_DISP_COUNT atexit dump for direct compare.
        let buckets = mlx_native::pipeline_dispatch_buckets();
        if !buckets.is_empty() {
            let total: u64 = buckets.iter().map(|(_, c)| *c).sum();
            eprintln!(
                "[MLX_DISP_BUCKET] Per-pipeline breakdown ({} unique pipelines, total={}):",
                buckets.len(),
                total,
            );
            for (label, count) in &buckets {
                let pct = if total > 0 {
                    100.0 * (*count as f64) / (total as f64)
                } else {
                    0.0
                };
                eprintln!(
                    "[MLX_DISP_BUCKET]   {:>8}  ({:5.2}%)  {}",
                    count, pct, label,
                );
            }
        }
    }

    Ok(())
}

// ================================================================
// Qwen3.5 generate path (ADR-013 P13.1)
// ================================================================

const QWEN35_PREFILL_SWEEP_TEXT: &str =
    "The benchmark prompt describes a careful engineering investigation. \
     It repeats neutral facts about measuring model speed, preserving token \
     probabilities, checking coherence, and comparing current code against \
     peer implementations. ";

fn qwen35_sweep_token_count(tokenizer: &tokenizers::Tokenizer, text: &str) -> Result<usize> {
    let encoding = tokenizer
        .encode(text, false)
        .map_err(|e| anyhow::anyhow!("qwen35 prefill sweep tokenization failed: {e}"))?;
    Ok(encoding.get_ids().len())
}

fn qwen35_sweep_prompt(
    tokenizer: &tokenizers::Tokenizer,
    target_tokens: usize,
) -> Result<(String, Vec<u32>)> {
    let repeated = QWEN35_PREFILL_SWEEP_TEXT.repeat((target_tokens / 20) + 500);
    let ids = tokenizer
        .encode(repeated.as_str(), false)
        .map_err(|e| anyhow::anyhow!("qwen35 prefill sweep base tokenization failed: {e}"))?
        .get_ids()
        .to_vec();
    anyhow::ensure!(
        ids.len() >= target_tokens,
        "qwen35 prefill sweep base prompt produced only {} tokens for target {}",
        ids.len(),
        target_tokens,
    );

    let mut best_text = tokenizer
        .decode(&ids[..target_tokens], false)
        .map_err(|e| anyhow::anyhow!("qwen35 prefill sweep decode failed: {e}"))?;
    let mut best_n = qwen35_sweep_token_count(tokenizer, &best_text)?;
    if best_n == target_tokens {
        let enc = tokenizer
            .encode(best_text.as_str(), false)
            .map_err(|e| anyhow::anyhow!("qwen35 prefill sweep final tokenization failed: {e}"))?;
        return Ok((best_text, enc.get_ids().to_vec()));
    }

    let lo = target_tokens.saturating_sub(256).max(1);
    let hi = (target_tokens + 256).min(ids.len());
    for n_ids in lo..=hi {
        let text = tokenizer
            .decode(&ids[..n_ids], false)
            .map_err(|e| anyhow::anyhow!("qwen35 prefill sweep decode failed: {e}"))?;
        let n = qwen35_sweep_token_count(tokenizer, &text)?;
        if n == target_tokens {
            let enc = tokenizer.encode(text.as_str(), false).map_err(|e| {
                anyhow::anyhow!("qwen35 prefill sweep final tokenization failed: {e}")
            })?;
            return Ok((text, enc.get_ids().to_vec()));
        }
        if n.abs_diff(target_tokens) < best_n.abs_diff(target_tokens) {
            best_text = text;
            best_n = n;
        }
    }

    let enc = tokenizer
        .encode(best_text.as_str(), false)
        .map_err(|e| anyhow::anyhow!("qwen35 prefill sweep final tokenization failed: {e}"))?;
    Ok((best_text, enc.get_ids().to_vec()))
}

fn qwen35_positions(seq_len: usize) -> Vec<i32> {
    let mut flat = vec![0i32; 4 * seq_len];
    for axis in 0..4 {
        for t in 0..seq_len {
            flat[axis * seq_len + t] = t as i32;
        }
    }
    flat
}

fn maybe_run_qwen35_prefill_sweep(
    model: &crate::inference::models::qwen35::model::Qwen35Model,
    tokenizer: &tokenizers::Tokenizer,
) -> Result<bool> {
    let Ok(lengths_raw) = std::env::var("HF2Q_QWEN35_PREFILL_SWEEP") else {
        return Ok(false);
    };
    let lengths: Vec<usize> = lengths_raw
        .split(',')
        .filter(|s| !s.trim().is_empty())
        .map(|s| {
            s.trim().parse::<usize>().with_context(|| {
                format!("HF2Q_QWEN35_PREFILL_SWEEP contains non-integer length {s:?}")
            })
        })
        .collect::<Result<_>>()?;
    anyhow::ensure!(
        !lengths.is_empty(),
        "HF2Q_QWEN35_PREFILL_SWEEP must contain at least one length"
    );
    let trials = std::env::var("HF2Q_QWEN35_PREFILL_SWEEP_TRIALS")
        .ok()
        .map(|s| {
            s.parse::<usize>()
                .context("parse HF2Q_QWEN35_PREFILL_SWEEP_TRIALS")
        })
        .transpose()?
        .unwrap_or(3)
        .max(1);
    let warmups = std::env::var("HF2Q_QWEN35_PREFILL_SWEEP_WARMUPS")
        .ok()
        .map(|s| {
            s.parse::<usize>()
                .context("parse HF2Q_QWEN35_PREFILL_SWEEP_WARMUPS")
        })
        .transpose()?
        .unwrap_or(1);

    use crate::inference::models::qwen35::io_heads::greedy_argmax_last_token;
    use crate::inference::models::qwen35::kv_cache::HybridKvCache;
    use crate::serve::multi_seq_kv::SlotId;
    use mlx_native::MlxDevice;

    fn top_n_indices(values: &[f32], n: usize) -> Vec<usize> {
        let mut indexed: Vec<(usize, f32)> = values.iter().copied().enumerate().collect();
        indexed.sort_unstable_by(|a, b| b.1.total_cmp(&a.1));
        indexed.into_iter().take(n).map(|(idx, _)| idx).collect()
    }

    fn logit_row_metrics(a: &[f32], b: &[f32]) -> (f32, f64, usize) {
        let mut max_abs = 0.0f32;
        let mut dot = 0.0f64;
        let mut a2 = 0.0f64;
        let mut b2 = 0.0f64;
        for (&av, &bv) in a.iter().zip(b) {
            max_abs = max_abs.max((av - bv).abs());
            let af = av as f64;
            let bf = bv as f64;
            dot += af * bf;
            a2 += af * af;
            b2 += bf * bf;
        }
        let cosine = if a2 > 0.0 && b2 > 0.0 {
            dot / (a2.sqrt() * b2.sqrt())
        } else {
            f64::NAN
        };
        let top_a = top_n_indices(a, 10);
        let top_b = top_n_indices(b, 10);
        let overlap = top_a.iter().filter(|idx| top_b.contains(idx)).count();
        (max_abs, cosine, overlap)
    }

    let device = MlxDevice::new()
        .map_err(|e| anyhow::anyhow!("qwen35 prefill sweep MlxDevice::new: {e}"))?;
    println!(
        "{{\"event\":\"qwen35_prefill_sweep_start\",\"lengths\":{:?},\"warmups\":{},\"trials\":{}}}",
        lengths, warmups, trials
    );

    for target in lengths {
        let (_prompt, prompt_tokens) = qwen35_sweep_prompt(tokenizer, target)?;
        let prompt_len = prompt_tokens.len();
        let positions = qwen35_positions(prompt_len);
        let max_seq = (prompt_len + 65)
            .max(128)
            .min(model.cfg.max_position_embeddings as usize);

        for iteration in 0..(warmups + trials) {
            let phase = if iteration < warmups {
                "warmup"
            } else {
                "measure"
            };
            let trial = if phase == "warmup" {
                iteration
            } else {
                iteration - warmups
            };
            let mut kv_cache = HybridKvCache::new(&model.cfg, &device, max_seq as u32, 1)
                .context("qwen35 prefill sweep HybridKvCache::new")?;
            let t0 = std::time::Instant::now();
            let full_logits =
                std::env::var("HF2Q_QWEN35_PREFILL_SWEEP_FULL_LOGITS").as_deref() == Ok("1");
            let compare_full_last =
                std::env::var("HF2Q_QWEN35_PREFILL_SWEEP_COMPARE_FULL_LAST").as_deref() == Ok("1");
            let logits = if compare_full_last {
                let mut full_kv = HybridKvCache::new(&model.cfg, &device, max_seq as u32, 1)
                    .context("qwen35 prefill sweep compare full HybridKvCache::new")?;
                let full = model
                    .forward_gpu(&prompt_tokens, &positions, &mut full_kv, SlotId(0))
                    .context("qwen35 prefill sweep compare forward_gpu")?;
                let mut last_kv = HybridKvCache::new(&model.cfg, &device, max_seq as u32, 1)
                    .context("qwen35 prefill sweep compare last HybridKvCache::new")?;
                let last = model
                    .forward_gpu_last_logits(&prompt_tokens, &positions, &mut last_kv, SlotId(0))
                    .context("qwen35 prefill sweep compare forward_gpu_last_logits")?;
                let vocab = model.cfg.vocab_size as usize;
                anyhow::ensure!(
                    full.len() == prompt_len * vocab,
                    "qwen35 prefill sweep compare full logits.len()={} != expected {}",
                    full.len(),
                    prompt_len * vocab,
                );
                anyhow::ensure!(
                    last.len() == vocab,
                    "qwen35 prefill sweep compare last logits.len()={} != expected {}",
                    last.len(),
                    vocab,
                );
                let full_last = &full[full.len() - vocab..];
                let full_token = greedy_argmax_last_token(full_last, model.cfg.vocab_size);
                let last_token = greedy_argmax_last_token(&last, model.cfg.vocab_size);
                let (max_abs, cosine, top10_overlap) = logit_row_metrics(full_last, &last);
                println!(
                    "{{\"event\":\"qwen35_prefill_sweep_compare\",\"target_tokens\":{},\"actual_tokens\":{},\"phase\":\"{}\",\"iteration\":{},\"trial\":{},\"full_token\":{},\"last_token\":{},\"max_abs\":{:.9},\"cosine\":{:.12},\"top10_overlap\":{}}}",
                    target,
                    prompt_len,
                    phase,
                    iteration,
                    trial,
                    full_token,
                    last_token,
                    max_abs,
                    cosine,
                    top10_overlap,
                );
                last
            } else if full_logits {
                model
                    .forward_gpu(&prompt_tokens, &positions, &mut kv_cache, SlotId(0))
                    .context("qwen35 prefill sweep forward_gpu")?
            } else {
                model
                    .forward_gpu_last_logits(&prompt_tokens, &positions, &mut kv_cache, SlotId(0))
                    .context("qwen35 prefill sweep forward_gpu_last_logits")?
            };
            let elapsed = t0.elapsed();
            let vocab = model.cfg.vocab_size as usize;
            let expected_logits = if full_logits && !compare_full_last {
                prompt_len * vocab
            } else {
                vocab
            };
            anyhow::ensure!(
                logits.len() == expected_logits,
                "qwen35 prefill sweep logits.len()={} != expected {} (full_logits={})",
                logits.len(),
                expected_logits,
                full_logits,
            );
            let last_logits = &logits[logits.len() - vocab..];
            let first_token = greedy_argmax_last_token(last_logits, model.cfg.vocab_size);
            let ms = elapsed.as_secs_f64() * 1000.0;
            let tps = prompt_len as f64 / elapsed.as_secs_f64();
            println!(
                "{{\"event\":\"qwen35_prefill_sweep\",\"target_tokens\":{},\"actual_tokens\":{},\"phase\":\"{}\",\"iteration\":{},\"trial\":{},\"prefill_ms\":{:.3},\"prefill_tps\":{:.3},\"first_token\":{},\"output_head\":\"{}\"}}",
                target,
                prompt_len,
                phase,
                iteration,
                trial,
                ms,
                tps,
                first_token,
                if full_logits { "all" } else { "last" },
            );
        }
    }
    println!("{{\"event\":\"qwen35_prefill_sweep_end\"}}");
    Ok(true)
}

/// Generate subcommand dispatch for `qwen35` / `qwen35moe` GGUF architectures.
///
/// Full end-to-end generate loop with stateful KV / SSM state threading:
///
/// 1. Load model + tokenizer from GGUF.
/// 2. Render + tokenize the prompt via GGUF's `tokenizer.chat_template`.
/// 3. Allocate [`HybridKvCache`] for the session (max_seq_len = prompt_len +
///    max_tokens, capped to `max_position_embeddings`).
/// 4. **Prefill**: call `forward_gpu(prompt_tokens, positions, kv_cache)`.
///    DeltaNet SSM state is threaded through `kv_cache.linear_attn` slots.
/// 5. **Decode loop** for `max_tokens` steps:
///    - Sample next token (argmax, temp=0 greedy).
///    - Check EOS (GGUF `tokenizer.ggml.eos_token_id`; default 151645 for
///      Qwen3.5 / Qwen3.6 per HF tokenizer_config.json).
///    - Call `forward_gpu([token], [pos], kv_cache)` — kv_cache carries the
///      DeltaNet conv+recurrent state from the previous step.
/// 6. Print the canonical hf2q 4-line header then the generated text.
///
/// # State threading
///
/// DeltaNet (Gated DeltaNet) layers maintain a recurrent state and a conv
/// ring-buffer that must persist across decode steps. `forward_gpu` now reads
/// from and writes back to the `kv_cache.linear_attn` slots on every call.
/// For full-attention layers the SDPA is re-run from scratch on each decode
/// token (the KV-append incremental path is a future optimisation).
/// Detect a true consecutive-repetition loop in `decoded_tokens`.
///
/// Returns `Some((ngram_size, occurrences))` if the LAST `N × ngram_size`
/// tokens are exactly `N` consecutive copies of the same `ngram_size`-token
/// sequence, for any `ngram_size ∈ {8, 12, 16, 20, 24}` and `N ≥ 4`.
///
/// Why "true consecutive" not "non-overlapping in a 128-token window":
/// the prior heuristic (4 non-overlapping occurrences of the last-N-token
/// key anywhere in the last 128 tokens) was overly aggressive on legitimate
/// enumeration patterns. 2026-05-03 user report: a wedding-cake recipe
/// shopping list with bullets like
///   `*   **Buy:** 5 lbs SMBC ...`
///   `*   **Buy:** 3 cans condensed milk ...`
///   `*   **Buy:** 1 box of Greenery ...`
///   `*   **Buy:** 1 Cake Drum ...`
///   `*   **Buy:** 1`  ← stop fired here
/// triggered the old detector at 666 useful tokens because the 8-token
/// `*   **Buy:** 1` prefix happened to appear ≥4 times non-consecutively
/// in the window — even though every bullet's CONTENT was different and
/// the model was making real progress.
///
/// True loops (the 2026-05-02 French-Toast `"I should make sure the
/// response is accurate and up-to-date."` × 30+ case) are CONSECUTIVE
/// — the model emits the SAME sequence end-to-end with no intervening
/// content. The consecutive-only check fires on those and ignores
/// structured enumeration.
///
/// Algorithm: take the LAST `ngram_size` tokens as the key, then check
/// whether the immediately preceding `ngram_size` tokens equal the key,
/// and the ones before that, and so on for `REPEAT_MIN_OCCURRENCES` total
/// copies. If all match, report a true loop.
fn detect_greedy_repetition_loop(decoded_tokens: &[u32]) -> Option<(usize, usize)> {
    detect_greedy_repetition_loop_with_text(decoded_tokens, |_| String::new())
}

/// Count consecutive copies of `key` ending at the tail of `decoded_tokens`,
/// starting from a known floor of `min_confirmed` already-verified copies.
/// Walks backward block-by-block and stops at the first mismatch or stream
/// start. Returns the total observed count (≥ `min_confirmed`).
///
/// Surfaces a truthful count to callers — the user-facing log line
/// "repeated N times" needs to reflect the actual run length, not the
/// detector's internal threshold (3 or 8). When a model genuinely emits
/// 40 copies of an 8-token cycle, the stop reason should say so.
fn count_consecutive_cycle_copies(
    decoded_tokens: &[u32],
    key: &[u32],
    min_confirmed: usize,
) -> usize {
    let ngram = key.len();
    let n = decoded_tokens.len();
    let mut actual = min_confirmed;
    let mut probe_start = match n.checked_sub((min_confirmed + 1) * ngram) {
        Some(p) => p,
        None => return actual,
    };
    loop {
        if &decoded_tokens[probe_start..probe_start + ngram] != key {
            break;
        }
        actual += 1;
        probe_start = match probe_start.checked_sub(ngram) {
            Some(p) => p,
            None => break,
        };
    }
    actual
}

/// Smart loop detector. Same cycle-discovery as the basic variant, but
/// when a candidate cycle is found, the closure decodes it to text and
/// the rep-threshold scales with the cycle's "content character ratio":
/// punctuation/whitespace-dominant cycles (e.g. legitimate markdown
/// table separator `| :--- | :--- | :--- |`) need MORE reps before
/// bailing; alphanumeric-dominant cycles (real model loops like
/// "Miticidal Agent, Fungicidal Agent, ...") fire at the lower threshold.
///
/// 2026-05-16 (BUG-coherence iter): the Phase C detector at MIN=3 fires
/// on legitimate markdown table separators (3-column `| :--- | :--- |
/// :--- |` cycles 3 times) — observed live on probe_long after the
/// batched-prefill workaround restored coherence.  Raising the bar
/// uniformly to 4+ misses real 18-token text loops; instead, raise it
/// only when the cycle is structurally repetitive (low letter content).
///
/// Content threshold: `alpha_ratio < 0.30` → require 8 reps (vs default 3).
/// Empirically this catches markdown separators (`|`, `:`, `-`, ` ` only
/// → 0% letters) without breaking real-text loop detection (Miticidal
/// Agent etc. is ≥ 70% letters).
fn detect_greedy_repetition_loop_with_text<F>(
    decoded_tokens: &[u32],
    mut decode_cycle: F,
) -> Option<(usize, usize)>
where
    F: FnMut(&[u32]) -> String,
{
    const MIN_NGRAM: usize = 2;
    const MAX_NGRAM: usize = 64;
    const MIN_OCCURRENCES_CONTENT: usize = 3;
    const MIN_OCCURRENCES_STRUCTURAL: usize = 8;
    const STRUCTURAL_ALPHA_THRESHOLD: f32 = 0.30;

    let n = decoded_tokens.len();
    for ngram in MIN_NGRAM..=MAX_NGRAM {
        // First check whether the cycle repeats at all at the lower (content)
        // threshold.  If yes, then decide threshold based on cycle content.
        let need_content = ngram * MIN_OCCURRENCES_CONTENT;
        if n < need_content {
            continue;
        }
        let tail = &decoded_tokens[n - need_content..];
        let key = &tail[need_content - ngram..];
        let mut all_match = true;
        for i in 0..MIN_OCCURRENCES_CONTENT {
            let start = i * ngram;
            if &tail[start..start + ngram] != key {
                all_match = false;
                break;
            }
        }
        if !all_match {
            continue;
        }

        // Cycle confirmed at MIN_OCCURRENCES_CONTENT.  Classify content.
        let cycle_text = decode_cycle(key);
        let total_chars = cycle_text.chars().count() as f32;
        // When no decoder is supplied (e.g. unit tests or callers without
        // a tokenizer), default to content classification — fire at the
        // lower threshold so we don't miss real loops.  Only treat as
        // STRUCTURAL when we have actual decoded text AND its alphabetic
        // content is below the threshold.
        if total_chars == 0.0 {
            let actual =
                count_consecutive_cycle_copies(decoded_tokens, key, MIN_OCCURRENCES_CONTENT);
            return Some((ngram, actual));
        }
        let alpha = cycle_text.chars().filter(|c| c.is_alphabetic()).count() as f32;
        let alpha_ratio = alpha / total_chars;

        if alpha_ratio >= STRUCTURAL_ALPHA_THRESHOLD {
            // Content-bearing cycle (real text loop) — fire at the
            // lower threshold immediately.
            let actual =
                count_consecutive_cycle_copies(decoded_tokens, key, MIN_OCCURRENCES_CONTENT);
            return Some((ngram, actual));
        }

        // Structural / punctuation-dominant cycle (table separator, list
        // bullets, dash runs, etc.).  Require MIN_OCCURRENCES_STRUCTURAL
        // before bailing — these are usually legitimate output.
        let need_structural = ngram * MIN_OCCURRENCES_STRUCTURAL;
        if n < need_structural {
            continue;
        }
        let tail2 = &decoded_tokens[n - need_structural..];
        let mut all_match2 = true;
        for i in 0..MIN_OCCURRENCES_STRUCTURAL {
            let start = i * ngram;
            if &tail2[start..start + ngram] != key {
                all_match2 = false;
                break;
            }
        }
        if all_match2 {
            let actual =
                count_consecutive_cycle_copies(decoded_tokens, key, MIN_OCCURRENCES_STRUCTURAL);
            return Some((ngram, actual));
        }
    }
    None
}

/// Special-token text fragments that, if present in the cumulative decode of
/// **generated** tokens, indicate the model has either:
///
/// * emitted an end-of-turn marker as text rather than as the configured
///   integer EOS id (Qwen3.x distill / merge variants), OR
/// * ended a Phi-3-style turn via the `<|end|>` text fragment (observed
///   2026-05-02 on a Qwen3.x-arch GGUF whose model emitted Phi-3-style
///   markers as multi-token text).
///
/// Order in this array determines tie-breaks when more than one fragment is
/// present in the input (`Iterator::find` returns the first hit). The
/// surrounding decode loop in `cmd_generate_qwen35` does not depend on which
/// fragment matched — it just stops on Some.
// Text-fragment stop-strings for finetunes that emit special tokens as
// BPE-decomposed bytes instead of as proper integer ids. Matches the
// stop-string contract of llama.cpp's CLI (`-r` / `--reverse-prompt`).
//
// 2026-05-03 — switched the leading marker from `<|im_end|>` to
// `<|im_start|>user`. On thinking-capable Qwen3 finetunes (notably the
// abliterix-EGA-abliterated dwq48 file in this repo's models/), the
// model emits `<|im_end|>` as a SEGMENT separator BETWEEN the thinking
// trace and the actual answer:
//
//   [thinking]<|im_end|>\n<|im_start|>assistant\n[ANSWER]<|im_end|>\n<|im_start|>user\n[hallucinations]
//                       ^                                            ^
//                       segment break, NOT turn-end                   model's "I've finished my
//                                                                    assistant turn" signal
//
// Stopping on the first `<|im_end|>` (the previous default — matches a
// properly-trained Qwen3 checkpoint, which uses `<|im_end|>` ONLY at
// turn-end) cut generation off before the answer was emitted. The
// turn-end signal on this finetune is the NEXT-turn opener
// `<|im_start|>user` (the model is hallucinating the next user input).
// Stopping there preserves the full assistant response (thinking +
// segment markers + answer + closing `<|im_end|>`).
//
// `<|endoftext|>` and `<|end|>` cover other end-of-generation markers
// (some Phi-3 / GPT-2 era models emit these as bytes).
const SPECIAL_TOKEN_STOPS: &[&str] = &[
    "<|im_start|>user",
    "<|im_start|>system",
    "<|endoftext|>",
    "<|end|>",
];

/// Returns the first [`SPECIAL_TOKEN_STOPS`] fragment present in `generated_text`,
/// or `None` if generation should continue.
///
/// The input MUST be the cumulative decode of **assistant-generated** tokens
/// only (NOT the prompt). The prompt typically contains `<|im_start|>` /
/// `<|im_end|>` chat-template markers; scanning those would short-circuit
/// generation immediately on every model.
#[allow(dead_code)] // production callers route via qwen35_visible_generated_prefix → special_token_safe_prefix; helper retained for unit tests.
fn find_special_token_stop(generated_text: &str) -> Option<&'static str> {
    find_special_token_stop_pos(generated_text).map(|(_, marker)| marker)
}

fn find_special_token_stop_pos(generated_text: &str) -> Option<(usize, &'static str)> {
    SPECIAL_TOKEN_STOPS
        .iter()
        .copied()
        .filter_map(|m| generated_text.find(m).map(|pos| (pos, m)))
        .min_by_key(|(pos, _)| *pos)
}

/// Decide whether the qwen3.5 generate-CLI loop needs the sampler-path
/// (forward_gpu_last_logits + sample_qwen35_logits_for_generate) or can take
/// the GPU-argmax fast-path (forward_gpu_greedy).
///
/// Mirrors llama.cpp's `--temp 0` semantics: when temperature is at or below
/// the deterministic floor, the argmax of the unwarped logits is the output —
/// top_k / top_p / min_p truncation is a no-op against an argmax-of-1. The
/// sampler-path is only needed when temperature scales the distribution OR
/// when repetition_penalty re-weights it (rep_penalty modifies logits before
/// argmax, so it cannot ride the GPU argmax fast-path).
///
/// Pre-2026-05-03 this returned `true` whenever any of top_k/top_p/min_p were
/// at their non-disabled defaults (40 / 0.95 / 0.05) regardless of temperature.
/// That meant `--temperature 0` still routed through the slower sampler path
/// (forward_gpu_last_logits downloads ~600KB of vocab logits to host;
/// forward_gpu_greedy returns 4 bytes after a GPU argmax kernel). On
/// qwen3.6-35B-A3B-dwq48 that cost ~19 tok/s vs the truly-greedy path at
/// 122 tok/s — the regression that matched 4adf689's pre-CLI-sampling perf.
fn qwen35_generate_uses_sampling(args: &cli::GenerateArgs) -> bool {
    args.temperature > crate::serve::sampler_pure::SAMPLING_EPS || args.repetition_penalty != 1.0
}

fn sample_qwen35_logits_for_generate(
    logits: &mut [f32],
    args: &cli::GenerateArgs,
    previous_tokens: &[u32],
) -> u32 {
    let params = crate::serve::sampler_pure::SamplingParams {
        temperature: args.temperature,
        top_p: args.top_p,
        top_k: args.top_k,
        min_p: args.min_p,
        repetition_penalty: args.repetition_penalty,
        max_tokens: args.max_tokens,
    };
    crate::serve::sampler_pure::sample_token(logits, &params, previous_tokens)
}

// ============================================================================
// Decode-loop testable helper (H3 Layer A unlock, item (c) of dossier plan)
// ============================================================================
//
// 2026-05-02 — extracts the qwen3.5 greedy-decode loop body from
// `cmd_generate_qwen35` into a pure helper that injects the model call as a
// closure. Lets unit tests feed synthetic token streams through the entire
// loop (including stop-condition + cumulative-decode + repetition-guard
// composition) without needing a Metal GPU. Closes the H3 finding from
// `docs/research/decode-test-gap-2026-05-02.md`: today's 12+4 helper tests
// pin individual pure functions in isolation but not the loop body that
// composes them.

/// Reason the decode loop stopped emitting tokens.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum DecodeStopReason {
    /// `next_token` matched one of `eos_token_ids` (integer-id stop).
    /// The matched id is included for diagnostics.
    EosTokenId(u32),
    /// Cumulative decode contained a [`SPECIAL_TOKEN_STOPS`] substring
    /// (text-fragment stop — covers tokenizers that emit special tokens
    /// as multi-token text instead of as the reserved id).
    SpecialTokenLeak(&'static str),
    /// `detect_greedy_repetition_loop` fired — greedy decode entered a
    /// stuck cycle.
    RepetitionLoop { ngram: usize, repeats: usize },
    /// KV cache would overrun on the next step.
    MaxSeq,
    /// `max_tokens` step budget exhausted without any of the above firing.
    MaxTokensReached,
}

/// Outcome of [`run_decode_loop`].
#[derive(Debug)]
pub(crate) struct DecodeLoopOutcome {
    /// Total number of tokens generated (including the seed token from
    /// prefill argmax).
    pub generated: usize,
    /// Why the loop stopped.
    pub stop_reason: DecodeStopReason,
}

/// One per-step event emitted to the `on_event` callback.
///
/// `step`, `token`, `cumulative_text` are exposed for unit-test
/// observation — production callers (cmd_generate_qwen35) only consume
/// `delta` for the streaming print, but tests assert on all four fields
/// to verify the loop state matches expectations.
#[allow(dead_code)]
pub(crate) struct DecodeStepEvent<'a> {
    /// Step index (0 is the seed token from prefill argmax; subsequent
    /// steps are 1..max_tokens, matching the production loop bounds).
    pub step: usize,
    /// The token just sampled by the model (or the seed token at step 0).
    pub token: u32,
    /// Cumulative-decoded text (covers all tokens generated so far,
    /// including the seed token).
    pub cumulative_text: &'a str,
    /// New byte-delta to emit since the previous step (the part of
    /// `cumulative_text` that wasn't in the previous step's emit).
    /// Empty if the new tokens didn't extend the cumulative text
    /// (e.g. a tokenizer rewrite changed an earlier prefix; we
    /// silently skip such cases for streaming correctness).
    pub delta: &'a str,
}

/// Run the qwen3.5 greedy-decode loop with injected forward + decode
/// closures.
///
/// The helper owns the per-step bookkeeping (cumulative-decode tracking,
/// stop checks, repetition guard, position arithmetic) and calls the
/// caller's `step_model` closure for each forward pass. This lets unit
/// tests feed a synthetic `step_model` that returns a pre-arranged token
/// stream, completely bypassing the GPU.
///
/// # Parameters
///
/// * `initial_token` — the token sampled from prefill argmax (becomes
///   step 0 of the cumulative decode).
/// * `prompt_len` — number of tokens in the (already-prefilled) prompt;
///   used to compute absolute positions.
/// * `max_seq` — KV-cache capacity. The loop stops with
///   [`DecodeStopReason::MaxSeq`] when the next position would equal or
///   exceed this value.
/// * `max_tokens` — step budget. `step=1..max_tokens` means at most
///   `max_tokens - 1` forward calls (matches the production loop bounds).
/// * `eos_token_ids` — integer EOS ids; matched against `next_token` at
///   the TOP of each iteration (mirrors production loop semantics).
/// * `step_model(prev_token, pos) -> Result<u32>` — produces the next
///   token. In production, this calls `model.forward_gpu_greedy(...)`.
///   In tests, it can return canned tokens.
/// * `decode_text(&[u32]) -> String` — decodes the cumulative token
///   list. In production, this is `tokenizer.decode(_, false)`.
/// * `on_event(DecodeStepEvent)` — invoked once per step AFTER the
///   forward call completes, with the cumulative text + delta.
///   Production binding prints the delta to stdout.
///
/// # Returns
///
/// [`DecodeLoopOutcome`] with the total tokens generated + stop reason.
///
/// # Errors
///
/// Propagates the first error from `step_model`. Does NOT propagate
/// errors from `decode_text` — a tokenizer decode failure is treated
/// as an empty string for that step (matches production behavior:
/// `unwrap_or_default()` at the call site).
pub(crate) fn run_decode_loop<F, P, E>(
    initial_token: u32,
    prompt_len: usize,
    max_seq: usize,
    max_tokens: usize,
    eos_token_ids: &[u32],
    mut step_model: F,
    mut decode_text: P,
    mut on_event: E,
) -> Result<DecodeLoopOutcome>
where
    F: FnMut(u32, i32, &[u32]) -> Result<u32>,
    P: FnMut(&[u32]) -> String,
    E: FnMut(DecodeStepEvent<'_>),
{
    let mut decoded_tokens: Vec<u32> = vec![initial_token];
    let mut emitted_text: String = String::new();
    let mut next_token = initial_token;
    let mut generated = 1usize;

    // Emit step 0 event (the seed token from prefill).
    //
    // 2026-05-03 — match llama.cpp's libllama philosophy: inference emits
    // raw decoded tokens. NO output-text scanning, NO visible-prefix
    // surgery, NO special-token-fragment scrubbing. The caller (CLI / SSE
    // server / harness) is the right layer to decide what to display, what
    // to truncate, and when to stop on application-level markers.
    //
    // Hard stops kept here (model-agnostic, layer-correct):
    //   * integer EOS via `eos_token_ids`
    //   * max_tokens budget
    //   * max_seq KV-cache overrun
    //   * consecutive n-gram repetition guard (catches genuine model loops
    //     that have NO clean stop signal — e.g. broken finetunes that
    //     never emit id 151645)
    //
    // Pre-2026-05-03 this loop body called `qwen35_visible_generated_prefix`
    // which hid `<|im_start|>` literal text + counted special-token-fragment
    // stops + did trailing-prefix-of-marker holding. For finetunes that
    // emit special tokens as BPE-decomposed bytes (this dwq48 file), that
    // text-fragment scaffolding stopped on the FIRST `<|im_end|>` text
    // appearance — which on simple prompts ("Hi", "Tell me a joke") is the
    // end-of-thinking marker, NOT the end-of-turn marker. The actual
    // answer comes between two `<|im_end|>` text fragments. Our stop fired
    // before the answer was emitted. Removed the inference-layer scanning
    // entirely; if a caller wants to truncate at `<|im_end|>` for display,
    // it does so as a display-layer post-process.
    let seed_full = decode_text(&decoded_tokens);
    on_event(DecodeStepEvent {
        step: 0,
        token: initial_token,
        cumulative_text: &seed_full,
        delta: &seed_full,
    });
    emitted_text.push_str(&seed_full);

    for step in 1..max_tokens {
        // Top-of-iteration integer-EOS check (mirrors production: line
        // 1646). Note: this catches the seed token if it's already EOS,
        // which preserves the production loop's `for step in 1..` shape.
        if eos_token_ids.contains(&next_token) {
            return Ok(DecodeLoopOutcome {
                generated,
                stop_reason: DecodeStopReason::EosTokenId(next_token),
            });
        }

        // Position arithmetic: step=1 maps to pos=prompt_len (the next
        // slot after prefill); step=N maps to pos=prompt_len + (N - 1).
        let pos = (prompt_len + step - 1) as i32;
        if (pos as usize) >= max_seq {
            return Ok(DecodeLoopOutcome {
                generated,
                stop_reason: DecodeStopReason::MaxSeq,
            });
        }

        // Forward pass.
        next_token = step_model(next_token, pos, &decoded_tokens)?;
        generated += 1;
        decoded_tokens.push(next_token);

        // Cumulative decode + delta computation.
        // 2026-05-03 — emit raw decoded text directly. See seed-event
        // comment above for the rationale (inference vs display layering).
        let new_full = decode_text(&decoded_tokens);
        let delta = if new_full.len() > emitted_text.len() && new_full.starts_with(&emitted_text) {
            &new_full[emitted_text.len()..]
        } else {
            // Tokenizer rewrote an earlier prefix (rare; happens with some
            // BPE merges) — emit nothing this step.
            ""
        };

        on_event(DecodeStepEvent {
            step,
            token: next_token,
            cumulative_text: &new_full,
            delta,
        });

        // Text-fragment EOS for finetunes that emit `<|im_end|>` (etc.) as
        // BPE-decomposed bytes instead of as their proper integer special-
        // token id. llama.cpp's CLI handles this via `--reverse-prompt` /
        // stop-strings as an application-layer gate; we implement the same
        // contract inline, scanning the cumulative decoded text for any
        // entry in `SPECIAL_TOKEN_STOPS`. This is NOT visible-text surgery
        // (we do not strip / re-format / hold-back trailing prefixes — the
        // delta the caller printed at this step is the raw model output).
        // The stop is a hard cut: raw display up to the boundary, then
        // exit with `SpecialTokenLeak`. Properly-trained models hit
        // integer-EOS first (above) and never trip this fallback.
        if let Some(marker) = find_special_token_stop(&new_full) {
            return Ok(DecodeLoopOutcome {
                generated,
                stop_reason: DecodeStopReason::SpecialTokenLeak(marker),
            });
        }

        // N-gram repetition guard.
        // Use the text-aware variant so legitimate structural repetition
        // (markdown table separators, list bullets, dash runs) requires
        // more reps before bailing than alphanumeric-content loops.
        if let Some((ngram, repeats)) =
            detect_greedy_repetition_loop_with_text(&decoded_tokens, |cycle| decode_text(cycle))
        {
            return Ok(DecodeLoopOutcome {
                generated,
                stop_reason: DecodeStopReason::RepetitionLoop { ngram, repeats },
            });
        }

        emitted_text = new_full;
    }

    Ok(DecodeLoopOutcome {
        generated,
        stop_reason: DecodeStopReason::MaxTokensReached,
    })
}

/// Streaming display-layer chat-template tag scrubber for the qwen3.5/3.6
/// CLI generate path.
///
/// Inference (`run_decode_loop`) emits raw decoded tokens — including
/// any literal `<|im_end|>` / `<|im_start|>{role}` chat-template tags
/// that the model produces as BPE-decomposed bytes (notably on the
/// abliterix-EGA-abliterated dwq48 finetune which doesn't emit these
/// as their proper integer special-token ids).  This struct sits at
/// the **display** layer between `run_decode_loop`'s on_event and
/// the user's terminal, suppressing those tags from the streamed
/// print without otherwise transforming the model's output.
///
/// Implementation: each delta is appended to a small rolling buffer.
/// We print the prefix of the buffer that CANNOT be the start of any
/// known tag (i.e. holds back the trailing N bytes if they could be
/// a prefix of `<|im_start|>system` etc.).  When the held-back bytes
/// resolve into a complete tag, we drop them; otherwise (the model
/// just emitted a `<` mid-text) we eventually print them out.
struct DisplayScrubber {
    buf: String,
    emitted: usize,
}

// Display-layer scrub list. These are STANDARD Qwen3 chat-template tokens
// that appear in our rendered prompt and that the model may regurgitate as
// BPE-decomposed bytes on broken finetunes. NOT a list of arbitrary
// pattern matches — every tag here is part of the Qwen3 chat-template
// vocabulary documented in the model's tokenizer_config.json.
const SCRUB_TAGS: &[&str] = &[
    "<|im_end|>",
    "<|im_start|>assistant",
    "<|im_start|>user",
    "<|im_start|>system",
    "<|endoftext|>",
    "<|end|>",
    // Standard reasoning-block tags (Qwen3-Thinking, DeepSeek-R1 family).
    "</think>",
    "<think>",
];

impl DisplayScrubber {
    fn new() -> Self {
        Self {
            buf: String::new(),
            emitted: 0,
        }
    }

    /// Append `delta`, return the bytes safe to print right now (with
    /// known tags scrubbed). Called once per decode step.
    fn push(&mut self, delta: &str) -> String {
        if delta.is_empty() {
            return String::new();
        }
        self.buf.push_str(delta);
        // Determine the largest position N <= buf.len() such that
        // buf[N..] is NOT a non-trivial prefix of any tag in SCRUB_TAGS.
        // Bytes [emitted..N] are safe to print (after scrubbing complete
        // tags); bytes [N..] are held back.
        let trail_hold = max_trailing_tag_prefix_len(&self.buf);
        let safe_to = self.buf.len().saturating_sub(trail_hold);
        if safe_to <= self.emitted {
            return String::new();
        }
        let raw_segment = &self.buf[self.emitted..safe_to];
        // Strip any complete tags that fall ENTIRELY within this segment.
        let mut out = raw_segment.to_string();
        for tag in SCRUB_TAGS {
            if out.contains(tag) {
                out = out.replace(tag, "");
            }
        }
        self.emitted = safe_to;
        out
    }

    /// Return any final bytes after the last on_event call (held-back
    /// trailing tag-prefixes that didn't resolve into a complete tag).
    fn flush(&self) -> String {
        if self.emitted >= self.buf.len() {
            return String::new();
        }
        let mut out = self.buf[self.emitted..].to_string();
        for tag in SCRUB_TAGS {
            if out.contains(tag) {
                out = out.replace(tag, "");
            }
        }
        out
    }
}

/// Length of the longest trailing slice of `text` that is a non-empty
/// proper prefix of any tag in [`SCRUB_TAGS`]. 0 if none.
fn max_trailing_tag_prefix_len(text: &str) -> usize {
    let mut max = 0;
    for tag in SCRUB_TAGS {
        for len in 1..tag.len() {
            let prefix = &tag[..len];
            if text.ends_with(prefix) && len > max {
                max = len;
            }
        }
    }
    max
}

fn cmd_generate_qwen35(args: cli::GenerateArgs, gguf: mlx_native::gguf::GgufFile) -> Result<()> {
    use crate::inference::models::qwen35::io_heads::greedy_argmax_last_token;
    use crate::inference::models::qwen35::kv_cache::HybridKvCache;
    use crate::serve::api::engine_qwen35::Qwen35LoadedModel;
    use crate::serve::multi_seq_kv::SlotId;
    use mlx_native::MlxDevice;
    use std::io::Write;

    let model_path = &args.model;

    // ADR-018 C3 — unified load path.
    //
    // cmd_generate_qwen35 previously did manual GGUF + tokenizer + EOS
    // resolution (~60 LOC of duplicated logic against
    // `Qwen35LoadedModel::load`). Refactor: route through the
    // `*LoadedModel::load` constructor so CLI and SERVE share one load
    // surface. Behaviour-equivalent: the constructor opens the GGUF
    // (re-open from the dispatcher's metadata-only handle is fine —
    // page cache is warm), loads weights via mlx-native, resolves
    // tokenizer + chat template + EOS + provenance, and emits the same
    // TTY-aware `\r loading i/n layers` progress line via
    // `header::LoadProgress`.
    //
    // The post-load cmd_generate_qwen35 body still uses the original
    // `gguf` handle (passed in by `cmd_generate`'s arch detect) for
    // chat-template render and EOS-from-metadata fallback — those are
    // mmap reads against the same on-disk file and don't need a second
    // open. `LoadInfo`'s build path takes its own `&GgufFile`; we pass
    // the parameter handle.
    let stdout_is_tty = std::io::IsTerminal::is_terminal(&std::io::stdout());

    if INVESTIGATION_ENV.dump_rendered_prompt.is_some()
        || std::env::var("HF2Q_DEBUG_TOKENIZE_ONLY").as_deref() == Ok("1")
    {
        let tokenizer =
            crate::inference::models::qwen35::tokenizer::build_tokenizer_from_gguf(&gguf)
                .context("Qwen35 tokenizer build for prompt diagnostics")?;
        let prompt_text_raw = resolve_prompt(&args)?;
        let prompt_text = render_chat_template(&gguf, &args, Some(&tokenizer), &prompt_text_raw)?;

        if let Some(dump_path) = INVESTIGATION_ENV.dump_rendered_prompt.as_deref() {
            std::fs::write(dump_path, prompt_text.as_bytes()).with_context(|| {
                format!("HF2Q_DUMP_RENDERED_PROMPT: failed to write {dump_path}")
            })?;
            eprintln!(
                "HF2Q_DUMP_RENDERED_PROMPT: wrote {} bytes to {}",
                prompt_text.len(),
                dump_path
            );
        }

        if std::env::var("HF2Q_DEBUG_TOKENIZE_ONLY").as_deref() == Ok("1") {
            let prompt_tokens =
                tokenize_rendered_prompt_llama_style(&gguf, &tokenizer, &prompt_text)?;
            let id_str: Vec<String> = prompt_tokens.iter().map(|i| i.to_string()).collect();
            println!("TOKENIZE_DEBUG_IDS: {}", id_str.join(" "));
        }

        return Ok(());
    }

    let load_opts = api::engine::LoadOptions {
        model_path: model_path.clone(),
        tokenizer_path: args.tokenizer.clone(),
        config_path: args.config.clone(),
        // cmd_generate path: no DWQ overlay surface yet (cli::GenerateArgs
        // doesn't carry --dwq-overlay; only ServeArgs does).
        dwq_overlay_path: None,
        // ADR-027 Phase A iter-6b.2: cmd_generate(_qwen35) honors
        // HF2Q_KV_PERSIST so qwen35 cold-process LCP resume works on the
        // CLI. Empty / unset env → None → in-process-only.
        kv_persist_dir: std::env::var("HF2Q_KV_PERSIST")
            .ok()
            .filter(|s| !s.is_empty())
            .map(std::path::PathBuf::from),
    };
    let load_start = std::time::Instant::now();
    let loaded = Qwen35LoadedModel::load(&load_opts).context("Qwen35LoadedModel::load")?;
    let load_elapsed = load_start.elapsed();

    // ADR-018 C3: emit the unified 13-line load banner BEFORE prompt
    // rendering / prefill. Order matches cmd_generate (Gemma) and the
    // legacy `print_header_top` site.
    let info = <Qwen35LoadedModel as load_info::LoadInfoBuilder>::build_load_info(
        &loaded,
        &gguf,
        load_elapsed,
        None,
        false,
    );
    load_info::emit_tracing(&info);
    let mut stdout = std::io::stdout();
    load_info::print_banner(&info, &mut stdout, stdout_is_tty).context("print load banner")?;

    // Partial-move the loaded artifacts into local mutable bindings so
    // the rest of cmd_generate_qwen35 (prefill + decode) keeps the same
    // shape. `Qwen35LoadedModel` has no manual `Drop`; partial moves
    // are sound.
    let model = loaded.model;
    let tokenizer = loaded.tokenizer;
    // The legacy code path resolved EOS via `gguf.metadata_u32(...)
    // .unwrap_or(151645)` AND emitted a `tracing::info!("Qwen3.5 EOS
    // token id: {}", eos_token_id)`. The unified surface stores
    // `eos_token_ids: Vec<u32>` in the loaded struct (already lifted
    // from GGUF or defaulted) — read it directly. The trailing
    // `tracing::info!` is replaced by emit_tracing's `eos_token_ids`
    // structured field.
    // 2026-05-02: cmd_generate_qwen35 had a single-eos collapse here that let
    // `<|im_end|>` (151645) leak through whenever `<|endoftext|>` (151643) was
    // first in the Vec — caused decode loops to run past 1st turn-end (user
    // report: French Toast prompt produced ~30-line repetition + multiple
    // `<|im_end|>`/`<|im_start|>assistant` leaks). Use the full Vec.
    // The legacy `eos_token_id` single value is kept for the SpecDecode path
    // below which still takes `Option<u32>` (TODO: extend SpecDecode to take
    // a slice; tracked in a follow-up to avoid scope creep here).
    // ADR-028 iter-266: SpecDecode now takes the full eos set via
    // run_with_eos_set; the legacy single-id was retired (was the
    // root cause of MTP K1 running past <|im_end|> per iter-263..265).
    let eos_token_ids: Vec<u32> = if loaded.eos_token_ids.is_empty() {
        vec![151_645]
    } else {
        loaded.eos_token_ids.clone()
    };

    if maybe_run_qwen35_prefill_sweep(&model, &tokenizer)? {
        return Ok(());
    }

    // ---- Resolve + render prompt ----
    let prompt_text_raw = resolve_prompt(&args)?;
    let prompt_text = render_chat_template(&gguf, &args, Some(&tokenizer), &prompt_text_raw)?;

    // Keep the Qwen path inspectable with the same investigation knob as the
    // Gemma path. This must happen before tokenization/decode so prompt bugs
    // can be isolated without a successful generation.
    if let Some(dump_path) = INVESTIGATION_ENV.dump_rendered_prompt.as_deref() {
        std::fs::write(dump_path, prompt_text.as_bytes())
            .with_context(|| format!("HF2Q_DUMP_RENDERED_PROMPT: failed to write {dump_path}"))?;
        eprintln!(
            "HF2Q_DUMP_RENDERED_PROMPT: wrote {} bytes to {}",
            prompt_text.len(),
            dump_path
        );
        return Ok(());
    }

    // ---- Tokenize ----
    let prompt_tokens = tokenize_rendered_prompt_llama_style(&gguf, &tokenizer, &prompt_text)?;
    let prompt_len = prompt_tokens.len();
    tracing::info!("Qwen3.5: {} prompt tokens", prompt_len);

    // Gate-time tokenizer parity tooling. When `HF2Q_DEBUG_TOKENIZE_ONLY=1`,
    // print the encoded token IDs on stdout (one space-separated line
    // prefixed `TOKENIZE_DEBUG_IDS:`) and exit `Ok(())` before model
    // load. Used by `scripts/qwen35_tokenizer_parity.sh` to compare
    // against `llama-tokenize` on the same GGUF — the parity contract
    // for the GGUF-driven tokenizer per ADR-013 §"Sovereignty".
    if std::env::var("HF2Q_DEBUG_TOKENIZE_ONLY").as_deref() == Ok("1") {
        let id_str: Vec<String> = prompt_tokens.iter().map(|i| i.to_string()).collect();
        println!("TOKENIZE_DEBUG_IDS: {}", id_str.join(" "));
        return Ok(());
    }

    let max_seq = (prompt_len + args.max_tokens + 64)
        .max(128)
        .min(model.cfg.max_position_embeddings as usize);
    let sample_logits = qwen35_generate_uses_sampling(&args);

    // ADR-034 task #78 Step 4 (2026-05-21) — HF2Q_SPEC_DFLASH=1 opt-in.
    // When set, this branch loads the Qwen35 DFlash drafter from
    // HF2Q_DFLASH_DRAFTER_PATH (default
    // /opt/hf2q/models/dflash-drafters/z-lab__Qwen3.6-27B-DFlash),
    // runs dispatch_qwen35_dflash_generate, prints the decoded text,
    // and returns Ok(()) before reaching the standard SpecDecode
    // / per-token branch. Greedy accept-walk consistency with the
    // spec path's OWN batched-prefill verifier argmax is preserved
    // at temperature=0 (NOT byte-identity to single-token decode —
    // empirically falsified at HEAD 6ad9d04c; see ADR-034 G3 row
    // at line 1501 + qwen35_orchestrator.rs module doc).
    //
    // Empirical perf validated at HEAD 334008e9 2026-05-22 — see
    // ADR-034 Cell B 27B (0.77x of MTP K=1 greedy) + Cell B 35B-A3B
    // (0.31x of base) rows. Output is coherence-degraded at long
    // lengths (27B 128-tok run garbles; see the eprintln warning in
    // try_dispatch_qwen35_dflash_spec_decode). Default OFF preserves
    // production perf parity AND faithful base-autoregressive output.
    let mut model = model;
    if let Some(()) = crate::serve::spec_decode_cli::try_dispatch_qwen35_dflash_spec_decode(
        &mut model,
        &prompt_tokens,
        args.max_tokens,
        &eos_token_ids,
        args.ignore_eos,
        &tokenizer,
    )? {
        return Ok(());
    }

    if let Some(()) = crate::serve::spec_decode_cli::try_dispatch_qwen35_eagle3_spec_decode(
        &mut model,
        &prompt_tokens,
        args.max_tokens,
        &eos_token_ids,
        args.ignore_eos,
        &tokenizer,
    )? {
        return Ok(());
    }

    let spec_env = std::env::var("HF2Q_SPEC_DECODE").ok();
    let mut use_spec_decode = match spec_env.as_deref() {
        Some("0") => false,
        Some("1") => true,
        _ => !sample_logits && (args.speculative || model.mtp.is_some()),
    };
    if sample_logits && args.speculative {
        tracing::warn!(
            "Speculative decoding requested with sampling parameters; using sampler path"
        );
    }
    if use_spec_decode && model.mtp.is_none() {
        tracing::warn!(
            "Speculative decoding requested but this GGUF has no MTP weights; using greedy decode"
        );
        use_spec_decode = false;
    }

    // ADR-018 C3: the legacy 2-line `print_header_top` site that lived
    // here is gone — the unified 13-line load banner already rendered
    // at the top of `cmd_generate_qwen35` (the byte-equivalent earlier
    // call to `load_info::print_banner`). The local variables that
    // backed the legacy header (`backend_chip`, `model_name`,
    // `total_gb`, `n_layers`, `header_top`) carried no value past the
    // print site and were deleted to keep the load-fact surface in one
    // place. `stdout` and `stdout_is_tty` were the only locals used
    // downstream — both are bound up top now.

    if use_spec_decode {
        use crate::inference::models::qwen35::spec_decode::SpecDecode;
        tracing::info!("Qwen3.5 speculative decode enabled");
        // ADR-013 P19 H12: warm GPU cache before SpecDecode's internal prefill
        // timer starts (matches the greedy branch). Idempotent if SpecDecode
        // already calls `ensure_gpu_cache_primed` internally.
        model
            .ensure_gpu_cache_primed()
            .context("Qwen35Model::ensure_gpu_cache_primed (P19 H12 spec-decode warmup)")?;

        // ADR-005 1bNEW.0 — `--benchmark` 5-iter loop, median + P95.
        // SpecDecode allocates and owns its own KV cache per call, so
        // each iter is a fresh cold prefill+decode by construction.
        // Streaming output and prefill-header printing are suppressed
        // in benchmark mode (see gemma path for the design rationale).
        if args.benchmark {
            let mut prefill_tps_runs: Vec<f64> = Vec::with_capacity(BENCH_NUM_RUNS);
            let mut decode_tps_runs: Vec<f64> = Vec::with_capacity(BENCH_NUM_RUNS);
            let mut generated_per_run: Vec<usize> = Vec::with_capacity(BENCH_NUM_RUNS);
            let mut accept_pct_runs: Vec<f64> = Vec::with_capacity(BENCH_NUM_RUNS);
            // ADR-034 task #91 (2026-05-21) codex review #2 — benchmark
            // path was ignoring sampler. Now plumbed through so
            // `--benchmark --temperature 0.6` exercises MH.
            let bench_sampler = crate::inference::models::qwen35::spec_decode::SpecSampler::new(
                args.temperature as f32,
                0,
            );
            for run_idx in 0..BENCH_NUM_RUNS {
                let result = SpecDecode::run_with_sampler_eos_set(
                    &model,
                    &prompt_tokens,
                    args.max_tokens,
                    eos_token_ids.clone(),
                    max_seq as u32,
                    bench_sampler,
                )
                .context("qwen35 SpecDecode::run_with_sampler_eos_set (benchmark)")?;
                let prefill_tps = if result.stats.prefill_elapsed.as_secs_f64() > 0.0 {
                    prompt_len as f64 / result.stats.prefill_elapsed.as_secs_f64()
                } else {
                    0.0
                };
                let gen = result.tokens.len();
                let decode_tps = if result.stats.decode_elapsed.as_secs_f64() > 0.0 {
                    gen as f64 / result.stats.decode_elapsed.as_secs_f64()
                } else {
                    0.0
                };
                let accept_pct = result.stats.acceptance_rate_pct();
                eprintln!(
                    "  Run {}/{}: prefill {} tok in {:.2}s ({:.1} tok/s); decode {} tok in {:.2}s ({:.1} tok/s; accept {:.1}%)",
                    run_idx + 1,
                    BENCH_NUM_RUNS,
                    prompt_len,
                    result.stats.prefill_elapsed.as_secs_f64(),
                    prefill_tps,
                    gen,
                    result.stats.decode_elapsed.as_secs_f64(),
                    decode_tps,
                    accept_pct,
                );
                prefill_tps_runs.push(prefill_tps);
                decode_tps_runs.push(decode_tps);
                generated_per_run.push(gen);
                accept_pct_runs.push(accept_pct);
            }
            let median_accept = median_f64(&accept_pct_runs);
            print_benchmark_summary(
                model_path,
                prompt_len,
                &generated_per_run,
                Some(&prefill_tps_runs),
                &decode_tps_runs,
                &[("Spec accept %".to_string(), format!("{:.1}", median_accept))],
            );
            return Ok(());
        }

        // ADR-028 iter-266: pass the full eos_token_ids set (not just
        // first()) so the SpecDecode runner can match BOTH
        // `<|endoftext|>` AND `<|im_end|>` (qwen3 chat-template stop).
        // Previously single-eos let `<|im_end|>` slip past, causing
        // MTP K1 path to run to max_tokens (ADR-028 iter-263..265).
        //
        // ADR-034 task #91 (2026-05-21): plumb CLI --temperature through
        // to SpecDecode via SpecSampler. temp == 0.0 keeps greedy path
        // (byte-identical to pre-#91); temp > 0.0 engages Metropolis-
        // Hastings stochastic acceptance at K=1 BATCHED. Default seed=0
        // for reproducibility in stochastic mode.
        let sampler = crate::inference::models::qwen35::spec_decode::SpecSampler::new(
            args.temperature as f32,
            0,
        );
        // ADR-034 task #90 Step 4 codex audit (2026-05-21) — honor
        // --ignore-eos by passing an EMPTY eos set when ignore_eos=true.
        // Was previously bug: spec_decode path silently kept the EOS set
        // even at --ignore-eos, causing K=N generation to stop at
        // <|im_end|> after ~8-14 tokens. Mirrors the dispatch_dflash_cli
        // pattern at src/serve/spec_decode_cli.rs:123.
        let effective_eos_for_spec: Vec<u32> = if args.ignore_eos {
            Vec::new()
        } else {
            eos_token_ids.clone()
        };
        let result = SpecDecode::run_with_sampler_eos_set(
            &model,
            &prompt_tokens,
            args.max_tokens,
            effective_eos_for_spec,
            max_seq as u32,
            sampler,
        )
        .context("qwen35 SpecDecode::run_with_sampler_eos_set")?;

        let prefill_tok_s = if result.stats.prefill_elapsed.as_secs_f64() > 0.0 {
            prompt_len as f64 / result.stats.prefill_elapsed.as_secs_f64()
        } else {
            0.0
        };
        header::print_header_prefill(
            &mut stdout,
            &header::HeaderInfoPrefill {
                prefill_n: prompt_len,
                prefill_ms: result.stats.prefill_elapsed.as_secs_f64() * 1000.0,
                prefill_tok_s,
            },
            stdout_is_tty,
        )
        .context("print header prefill")?;

        let decoded = tokenizer.decode(&result.tokens, false).unwrap_or_default();
        print!("{}", decoded);
        stdout.flush()?;

        let generated = result.tokens.len();
        let tok_per_sec = if result.stats.decode_elapsed.as_secs_f64() > 0.0 {
            generated as f64 / result.stats.decode_elapsed.as_secs_f64()
        } else {
            0.0
        };
        let (td, tr) = if std::io::IsTerminal::is_terminal(&std::io::stderr()) {
            ("\x1b[2m", "\x1b[0m")
        } else {
            ("", "")
        };
        eprintln!(
            "\n\n{td}--- mlx-native (qwen35 spec): {} tokens in {:.2}s ({:.1} tok/s, accept {:.1}%) ---{tr}",
            generated,
            result.stats.decode_elapsed.as_secs_f64(),
            tok_per_sec,
            result.stats.acceptance_rate_pct(),
        );

        return Ok(());
    }

    // ---- Allocate HybridKvCache ----
    // ADR-028 iter-131: AUDIT FINDING — the CLI generate path was silently
    // ignoring HF2Q_TQ_KV=1 (used legacy `new` → tq_kv_active=false). This
    // turns out to be PERFORMANCE-PRESERVING at qwen3.6 (kv_heads=2):
    // measured TQ-HB engagement gives 115/106 t/s (short/long) vs the
    // F32-path 128/110 t/s — TQ-HB FWHT+encode overhead exceeds bandwidth
    // savings at small kv_heads scale. iter-131 leaves the legacy `new`
    // path in place to preserve the F32 perf, but documents the gap. To
    // engage TQ-HB explicitly (memory savings, slight perf cost), use the
    // engine API endpoint (POST /generate) which routes through
    // alloc_kv_cache_for_request → new_with_options.
    let device = MlxDevice::new().map_err(|e| anyhow::anyhow!("MlxDevice::new: {e}"))?;
    let mut kv_cache =
        HybridKvCache::new(&model.cfg, &device, max_seq as u32, 1).context("HybridKvCache::new")?;
    tracing::info!(
        "Qwen3.5 KV cache allocated: max_seq={}, {} MB",
        max_seq,
        kv_cache.total_bytes() / (1024 * 1024)
    );

    // ---- Prefill ----
    // Build flat positions [4 * prompt_len]: axis-major, all axes = token index.
    let prefill_positions: Vec<i32> = {
        let mut flat = vec![0i32; 4 * prompt_len];
        for axis in 0..4 {
            for t in 0..prompt_len {
                flat[axis * prompt_len + t] = t as i32;
            }
        }
        flat
    };

    // ADR-013 P19 H12 (2026-05-01): warm the per-thread GPU cache (one-shot
    // ~17 GB Q4 weight materialization onto Metal heap + lm_head BF16/Q4_0
    // pre-quant + flash_attn_prefill kernel registration) BEFORE the prefill
    // timer starts. Without this, the ~984 ms upload was charged to
    // `prefill_tok_s`, producing a 28× apparent gap vs llama.cpp's
    // `prompt eval time` (which excludes model load by construction).
    // Compute is unchanged; only the timer-span moves. `wave5b8_profile`'s
    // `UploadWeights` section still tracks the cost — it just lives here now.
    let warmup_start = std::time::Instant::now();
    model
        .ensure_gpu_cache_primed()
        .context("Qwen35Model::ensure_gpu_cache_primed (P19 H12 warmup)")?;
    tracing::info!(
        "Qwen3.5 GPU warmup (P19 H12): {:.2}s",
        warmup_start.elapsed().as_secs_f64()
    );

    // ADR-005 1bNEW.0 — `--benchmark` 5-iter loop, median + P95.
    // Each iter resets the HybridKvCache (zero cursors + zero K/V buffers
    // via `reset_all_buffers`) then re-runs prefill + decode. Warmup
    // already ran above (outside the per-iter timer — preserves the
    // ADR-013 P19 H12 timing semantics across all 5 iterations).
    //
    // Streaming output is suppressed: the `on_event` callback is a
    // no-op and the cumulative-text accumulator is unused (decode_text
    // closure returns "" so the special-token-leak detector never
    // fires). EOS / max-tokens / max-seq stops still work; the
    // n-gram repetition guard inside `run_decode_loop` is also active.
    if args.benchmark {
        let mut prefill_tps_runs: Vec<f64> = Vec::with_capacity(BENCH_NUM_RUNS);
        let mut decode_tps_runs: Vec<f64> = Vec::with_capacity(BENCH_NUM_RUNS);
        let mut generated_per_run: Vec<usize> = Vec::with_capacity(BENCH_NUM_RUNS);
        for run_idx in 0..BENCH_NUM_RUNS {
            kv_cache.reset_all_buffers();

            let prefill_start = std::time::Instant::now();
            let prefill_logits = model
                // ADR-040 Phase B4b (2026-05-24): CLI benchmark path is
                // single-seq (kv_cache allocated at n_seqs=1); slot 0
                // preserves pre-B4b behaviour.
                .forward_gpu_last_logits(
                    &prompt_tokens,
                    &prefill_positions,
                    &mut kv_cache,
                    SlotId(0),
                )
                .context("Qwen35Model::forward_gpu_last_logits (benchmark prefill)")?;
            let prefill_elapsed = prefill_start.elapsed();
            let prefill_tps = if prefill_elapsed.as_secs_f64() > 0.0 {
                prompt_len as f64 / prefill_elapsed.as_secs_f64()
            } else {
                0.0
            };

            let next_token = if sample_logits {
                let mut logits = prefill_logits.to_vec();
                sample_qwen35_logits_for_generate(&mut logits, &args, &[])
            } else {
                greedy_argmax_last_token(&prefill_logits, model.cfg.vocab_size)
            };

            let decode_start = std::time::Instant::now();
            let outcome = if sample_logits {
                run_decode_loop(
                    next_token,
                    prompt_len,
                    max_seq,
                    args.max_tokens,
                    &eos_token_ids,
                    |prev_token, pos, generated_tokens| -> Result<u32> {
                        let decode_positions = vec![pos; 4];
                        let mut logits = model
                            .forward_gpu_last_logits(
                                &[prev_token],
                                &decode_positions,
                                &mut kv_cache,
                                SlotId(0),
                            )
                            .with_context(|| {
                                format!("forward_gpu_last_logits decode at pos {pos} (benchmark)")
                            })?;
                        Ok(sample_qwen35_logits_for_generate(
                            &mut logits,
                            &args,
                            generated_tokens,
                        ))
                    },
                    |_toks| String::new(),
                    |_event| {},
                )?
            } else {
                run_decode_loop(
                    next_token,
                    prompt_len,
                    max_seq,
                    args.max_tokens,
                    &eos_token_ids,
                    |prev_token, pos, _generated_tokens| -> Result<u32> {
                        let decode_positions = vec![pos; 4];
                        model
                            // ADR-040 Phase B4d (2026-05-30) —
                            // forward_gpu_greedy now accepts a SlotId
                            // (was hard-coded SlotId(0) per B4d
                            // deferral). Production cmd_generate path
                            // is single-seq today (FifoSerial /
                            // SerialFifo); SlotId(0) is byte-identical
                            // to pre-B4d.
                            .forward_gpu_greedy(
                                &[prev_token],
                                &decode_positions,
                                &mut kv_cache,
                                SlotId(0),
                            )
                            .with_context(|| {
                                format!("forward_gpu_greedy decode at pos {pos} (benchmark)")
                            })
                    },
                    |_toks| String::new(),
                    |_event| {},
                )?
            };
            let decode_elapsed = decode_start.elapsed();
            let gen = outcome.generated;
            let decode_tps = if decode_elapsed.as_secs_f64() > 0.0 {
                gen as f64 / decode_elapsed.as_secs_f64()
            } else {
                0.0
            };
            eprintln!(
                "  Run {}/{}: prefill {} tok in {:.2}s ({:.1} tok/s); decode {} tok in {:.2}s ({:.1} tok/s)",
                run_idx + 1,
                BENCH_NUM_RUNS,
                prompt_len,
                prefill_elapsed.as_secs_f64(),
                prefill_tps,
                gen,
                decode_elapsed.as_secs_f64(),
                decode_tps,
            );
            prefill_tps_runs.push(prefill_tps);
            decode_tps_runs.push(decode_tps);
            generated_per_run.push(gen);
        }
        print_benchmark_summary(
            model_path,
            prompt_len,
            &generated_per_run,
            Some(&prefill_tps_runs),
            &decode_tps_runs,
            &[],
        );
        return Ok(());
    }

    tracing::info!("Qwen3.5 prefill: seq_len={}", prompt_len);
    // ADR-013 P19 H9 (2026-05-01): empirical CB-sync attribution.  Reset
    // mlx-native's atomic counters before prefill, read after, and print
    // when HF2Q_PROFILE_SYNC=1 is set.  This is a measurement-only diff:
    // SYNC_COUNT counts commit_and_wait calls/prefill, DISPATCH_COUNT
    // counts kernel dispatches, BARRIER_COUNT counts memory_barriers.
    // The hypothesis ("hf2q does ~120-160 commit_and_wait/prefill while
    // llama.cpp does ~1") is TESTABLE by comparing these numbers against
    // ggml-metal's per-graph submit pattern.  Zero overhead when env
    // unset (RAII guard does no work).
    let profile_sync = std::env::var("HF2Q_PROFILE_SYNC").is_ok();
    if profile_sync {
        mlx_native::reset_counters();
    }
    let prefill_start = std::time::Instant::now();
    let prefill_logits = model
        // ADR-040 Phase B4b (2026-05-24): CLI single-seq prefill, slot 0.
        .forward_gpu_last_logits(&prompt_tokens, &prefill_positions, &mut kv_cache, SlotId(0))
        .context("Qwen35Model::forward_gpu_last_logits (prefill)")?;
    let prefill_elapsed = prefill_start.elapsed();
    if profile_sync {
        eprintln!(
            "[P19 H9] prefill seq_len={} elapsed_ms={:.1} sync_count={} dispatch_count={} barrier_count={} cmd_buf_count={}",
            prompt_len,
            prefill_elapsed.as_secs_f64() * 1000.0,
            mlx_native::sync_count(),
            mlx_native::dispatch_count(),
            mlx_native::barrier_count(),
            mlx_native::cmd_buf_count(),
        );
    }

    // Sanity-check logits shape.
    let vocab_size = model.cfg.vocab_size;
    anyhow::ensure!(
        prefill_logits.len() == vocab_size as usize,
        "forward_gpu_last_logits (prefill) returned logits.len()={} != vocab({})",
        prefill_logits.len(),
        vocab_size,
    );

    let prefill_tok_s = prompt_len as f64 / prefill_elapsed.as_secs_f64();
    header::print_header_prefill(
        &mut stdout,
        &header::HeaderInfoPrefill {
            prefill_n: prompt_len,
            prefill_ms: prefill_elapsed.as_secs_f64() * 1000.0,
            prefill_tok_s,
        },
        stdout_is_tty,
    )
    .context("print header prefill")?;

    // HF2Q_DUMP_LOGITS=1: write the last-token logit vector to /tmp/hf2q_logits_t0.bin
    // and exit immediately. Used for first-token logit comparison vs llama.cpp.
    if std::env::var("HF2Q_DUMP_LOGITS").as_deref() == Ok("1") {
        let last_logits = &prefill_logits;
        let bytes: &[u8] = unsafe {
            std::slice::from_raw_parts(last_logits.as_ptr() as *const u8, last_logits.len() * 4)
        };
        std::fs::write("/tmp/hf2q_logits_t0.bin", bytes)
            .context("HF2Q_DUMP_LOGITS: write /tmp/hf2q_logits_t0.bin")?;
        // Top-3 to stderr for quick sanity check.
        let mut indexed: Vec<(usize, f32)> = last_logits.iter().copied().enumerate().collect();
        indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
        eprintln!(
            "HF2Q_DUMP_LOGITS: wrote {} f32 values to /tmp/hf2q_logits_t0.bin",
            last_logits.len()
        );
        eprintln!("  top-3: {:?}", &indexed[..3.min(indexed.len())]);
        return Ok(());
    }

    // Sample the first token from prefill logits (last token's row). This
    // mirrors llama.cpp's common_sampler contract: the CLI default sampling
    // parameters affect token 0, instead of silently downcasting generation to
    // greedy argmax.
    let last_prefill_logits = &prefill_logits;
    let next_token = if sample_logits {
        let mut logits = last_prefill_logits.to_vec();
        sample_qwen35_logits_for_generate(&mut logits, &args, &[])
    } else {
        greedy_argmax_last_token(last_prefill_logits, vocab_size)
    };
    tracing::info!("Qwen3.5 first decoded token: {}", next_token);

    // 2026-05-02: cumulative-decode + delta-print pattern (mirrors llama.cpp's
    // `tok_str_pos` discipline). Per-token `tokenizer.decode(&[t], ...)` breaks
    // multi-byte UTF-8 codepoints (emoji, CJK) at token boundaries → garble
    // like `���` in the user's French-Toast output. Decode the full so-far
    // sequence each step and print only the byte-delta. Costs O(generated)
    // bytes of allocation per step but produces correct UTF-8 every time.
    // Also catches special-token-string leaks (`<|im_end|>` etc. emitted as
    // text rather than as the eos_token_ids vocab id) — see is_special_token
    // check below.
    // Initial seed-token print is now handled inside the loop helper's
    // step=0 event (the helper takes care of the entire cumulative-decode
    // bookkeeping). We just bind the closures and fire it.

    // ---- Decode loop ----
    let decode_start = std::time::Instant::now();
    let step_profile_enabled = std::env::var("HF2Q_STEP_PROFILE").is_ok();

    // 2026-05-03 — split run_decode_loop dispatch into greedy and sampling
    // ARMS rather than branching `if sample_logits` per step inside the
    // step_model closure. The branched closure captured `sample_logits`,
    // `args`, AND `model` (sampling needed forward_gpu_last_logits +
    // sample_qwen35_logits_for_generate; greedy needed forward_gpu_greedy);
    // even though the branch was loop-invariant, the captured-state struct
    // size + the layered indirection (closure → if/else → method call)
    // measurably regressed greedy decode by ~19 tok/s (104 vs 122) on
    // qwen3.6-35B-A3B-dwq48 200-token greedy bench (HEAD@588ccb6 vs
    // 4adf689). Bisecting empirically: removing only the branch from the
    // closure body restored 122 tok/s exactly; reverting decode_stream or
    // qwen35_visible_generated_prefix did NOT recover the gap. Two-arm
    // dispatch monomorphizes each path independently, eliminating the
    // per-step branch + indirection.
    //
    // Cumulative-decode strategy: both arms use a stateful HuggingFace
    // `decode_stream` (O(1) per token). The non-stream alternative —
    // `tokenizer.decode(toks, false)` — re-tokenizes O(N) per call and
    // sum-O(N²) per generation; that pattern showed 60 tok/s on a
    // 1832-token sampling run (vs 116 tok/s peer). decode_stream + the
    // `cumulative_text.push_str` accumulator reproduces the same
    // `&str` value run_decode_loop expects without the quadratic blowup.
    // Display-layer chat-template tag scrubber. Inference emits raw
    // bytes; this filter sits on the streaming-print path and suppresses
    // literal `<|im_end|>` / `<|im_start|>{role}` tags from the user's
    // terminal without touching the inference layer.
    let mut display = DisplayScrubber::new();

    let outcome = if sample_logits {
        let mut decode_stream = tokenizer.decode_stream(false);
        let mut cumulative_text = String::new();
        let mut last_decoded_count = 1usize;
        if let Ok(Some(s)) = decode_stream.step(next_token) {
            cumulative_text.push_str(&s);
        }

        run_decode_loop(
            next_token,
            prompt_len,
            max_seq,
            args.max_tokens,
            &eos_token_ids,
            |prev_token, pos, generated_tokens| -> Result<u32> {
                let decode_positions = vec![pos; 4];
                let _t_step = step_profile_enabled.then(std::time::Instant::now);
                let _t_fwd = step_profile_enabled.then(std::time::Instant::now);
                let mut logits = model
                    .forward_gpu_last_logits(
                        &[prev_token],
                        &decode_positions,
                        &mut kv_cache,
                        SlotId(0),
                    )
                    .with_context(|| format!("forward_gpu_last_logits decode at pos {pos}"))?;
                let fwd_us = _t_fwd.map(|t| t.elapsed().as_micros());
                let _t_smp = step_profile_enabled.then(std::time::Instant::now);
                let next = sample_qwen35_logits_for_generate(&mut logits, &args, generated_tokens);
                let smp_us = _t_smp.map(|t| t.elapsed().as_micros());
                if let Some(t) = _t_step {
                    eprintln!(
                        "[STEP_PROFILE] pos={pos} total={:.2}ms fwd={:.2}ms smp={:.2}ms",
                        t.elapsed().as_micros() as f64 / 1000.0,
                        fwd_us.unwrap_or(0) as f64 / 1000.0,
                        smp_us.unwrap_or(0) as f64 / 1000.0,
                    );
                }
                Ok(next)
            },
            |toks| {
                while last_decoded_count < toks.len() {
                    let tok = toks[last_decoded_count];
                    if let Ok(Some(s)) = decode_stream.step(tok) {
                        cumulative_text.push_str(&s);
                    }
                    last_decoded_count += 1;
                }
                cumulative_text.clone()
            },
            |event| {
                let cleaned = display.push(event.delta);
                if !cleaned.is_empty() {
                    print!("{}", cleaned);
                    let _ = stdout.flush();
                }
            },
        )?
    } else {
        let mut decode_stream = tokenizer.decode_stream(false);
        let mut cumulative_text = String::new();
        let mut last_decoded_count = 1usize;
        if let Ok(Some(s)) = decode_stream.step(next_token) {
            cumulative_text.push_str(&s);
        }

        run_decode_loop(
            next_token,
            prompt_len,
            max_seq,
            args.max_tokens,
            &eos_token_ids,
            |prev_token, pos, _generated_tokens| -> Result<u32> {
                let decode_positions = vec![pos; 4];
                let _t_step = step_profile_enabled.then(std::time::Instant::now);
                // forward_gpu_greedy: GPU argmax → 4-byte download (vs 600KB full logits).
                // Eliminates ~5ms/token vocabulary download for greedy decode.
                let next = model
                    // ADR-040 Phase B4d (2026-05-30) — see sibling at
                    // serve/mod.rs:3184 for the SlotId contract.
                    .forward_gpu_greedy(&[prev_token], &decode_positions, &mut kv_cache, SlotId(0))
                    .with_context(|| format!("forward_gpu_greedy decode at pos {pos}"))?;
                if let Some(t) = _t_step {
                    eprintln!(
                        "[STEP_PROFILE] pos={pos} total={:.2}ms",
                        t.elapsed().as_micros() as f64 / 1000.0
                    );
                }
                Ok(next)
            },
            |toks| {
                while last_decoded_count < toks.len() {
                    let tok = toks[last_decoded_count];
                    if let Ok(Some(s)) = decode_stream.step(tok) {
                        cumulative_text.push_str(&s);
                    }
                    last_decoded_count += 1;
                }
                cumulative_text.clone()
            },
            |event| {
                let cleaned = display.push(event.delta);
                if !cleaned.is_empty() {
                    print!("{}", cleaned);
                    let _ = stdout.flush();
                }
            },
        )?
    };

    // Flush any held-back trailing bytes from the display scrubber. These
    // are bytes that COULD have been the start of a chat-template tag at
    // the time on_event was called but never resolved into one before the
    // loop ended.
    let trailing = display.flush();
    if !trailing.is_empty() {
        print!("{}", trailing);
        let _ = stdout.flush();
    }

    // Loop-side stop reasons get a one-line tracing emit + (for repetition)
    // a stderr diagnostic, matching the production behavior the loop
    // previously emitted inline. EosTokenId / MaxTokensReached are the
    // expected exits; MaxSeq is informational; the rest are loud.
    match &outcome.stop_reason {
        DecodeStopReason::EosTokenId(_) | DecodeStopReason::MaxTokensReached => {}
        DecodeStopReason::MaxSeq => {
            tracing::warn!(
                "Qwen3.5 decode: reached max_seq {} after {} tokens; stopping",
                max_seq,
                outcome.generated
            );
        }
        DecodeStopReason::SpecialTokenLeak(marker) => {
            tracing::info!(
                "Qwen3.5 decode: special-token string `{marker}` detected in \
                 cumulative decode after {} tokens; stopping.",
                outcome.generated
            );
        }
        DecodeStopReason::RepetitionLoop { ngram, repeats } => {
            tracing::info!(
                "Qwen3.5 decode: consecutive n-gram repetition detected after {} \
                 tokens (last {} tokens repeated {} times consecutively); stopping.",
                outcome.generated,
                ngram,
                repeats
            );
            // 2026-05-03 — when the user passed `--no-thinking` AND the loop
            // fired EARLY (within 32-200 tokens), the most likely cause is
            // the empty-`<think>\n\n</think>\n\n` suppressor in the chat
            // template confusing the model into a degenerate attractor on
            // this specific prompt. Both hf2q and llama.cpp exhibit this
            // failure mode with `--no-thinking` on Qwen3.5/3.6 thinking-
            // capable checkpoints (verified ADR-005 morning session). The
            // canonical fix is to drop `--no-thinking` and let the
            // auto-detected default (thinking-on) handle it; the model
            // will emit an internal reasoning trace then the answer.
            if args.no_thinking && outcome.generated < 200 {
                eprintln!(
                    "\n[hf2q] decode entered a {}-token consecutive-repetition loop \
                     after only {} tokens with `--no-thinking`. On this Qwen3.5/3.6 \
                     thinking-capable checkpoint, the empty `<think></think>` \
                     suppressor is a known degenerate attractor for some prompts \
                     (both hf2q AND llama.cpp produce the same loop). \
                     Recommended fix: drop `--no-thinking` and let the auto-\
                     detected thinking-mode handle it (the model will emit a \
                     reasoning trace, then the answer). \
                     Alternative escape: raise `--repetition-penalty` or use \
                     `--temperature` >0.",
                    ngram, outcome.generated
                );
            } else {
                eprintln!(
                    "\n[hf2q] decode entered a {}-token consecutive-repetition loop \
                     after {} tokens — stopping. The model emitted the same \
                     {}-token sequence {} times in a row; raise --repetition-penalty \
                     or use a higher --temperature / lower --min-p to escape.",
                    ngram, outcome.generated, ngram, repeats
                );
            }
        }
    }

    let generated = outcome.generated;
    let decode_elapsed = decode_start.elapsed();
    let tok_per_sec = generated as f64 / decode_elapsed.as_secs_f64();

    let (td, tr) = if std::io::IsTerminal::is_terminal(&std::io::stderr()) {
        ("\x1b[2m", "\x1b[0m")
    } else {
        ("", "")
    };
    eprintln!(
        "\n\n{td}--- mlx-native (qwen35): {} tokens in {:.2}s ({:.1} tok/s) ---{tr}",
        generated,
        decode_elapsed.as_secs_f64(),
        tok_per_sec,
    );

    // `--benchmark` is handled by the 5-run loop earlier in this
    // function (which `return Ok(())`s before reaching here); the
    // single-run summary block that lived here pre-restoration is
    // unreachable when `args.benchmark` is true. See
    // `if args.benchmark { ... }` above (qwen35 normal-decode branch).

    // ADR-028 iter-290: qwen35 generate path mirror of HF2Q_DUMP_COUNTERS dump
    // from gemma4's cmd_generate (at /opt/hf2q/src/serve/mod.rs:1361).
    // Enables cross-model per-pipeline comparison.
    if std::env::var("HF2Q_DUMP_COUNTERS").ok().as_deref() == Some("1") {
        let dispatches = mlx_native::dispatch_count();
        let syncs = mlx_native::sync_count();
        eprintln!(
            "[MLX_COUNTERS] dispatches={} syncs={} prompt_tokens={} decode_tokens={}",
            dispatches, syncs, prompt_len, generated,
        );
        let buckets = mlx_native::pipeline_dispatch_buckets();
        if !buckets.is_empty() {
            let total: u64 = buckets.iter().map(|(_, c)| *c).sum();
            eprintln!(
                "[MLX_DISP_BUCKET] Per-pipeline breakdown ({} unique pipelines, total={}):",
                buckets.len(),
                total,
            );
            for (label, count) in &buckets {
                let pct = if total > 0 {
                    100.0 * (*count as f64) / (total as f64)
                } else {
                    0.0
                };
                eprintln!(
                    "[MLX_DISP_BUCKET]   {:>8}  ({:5.2}%)  {}",
                    count, pct, label,
                );
            }
        }
    }

    Ok(())
}

/// ADR-005 Phase 4 iter-208 (W76): callable engine loader extracted from
/// `cmd_serve` so the [`multi_model::HotSwapManager`] can dispatch against
/// the same load path the single-model startup uses.
///
/// **Pure refactor.**  The body of this function is the byte-identical
/// sequence that previously lived inline in `cmd_serve` (header validate
/// → `LoadedModel::load` → `Engine::spawn` → optional synchronous warmup
/// on a one-shot tokio runtime).  `cmd_serve` now calls this for the
/// existing single-model startup path; iter-209 will plumb it through
/// `HotSwapManager::load_or_get` so a second model load mid-process is
/// possible without re-implementing the load.
///
/// **Why synchronous warmup.**  Iter-103 (`8109954`) fixed the
/// chat-warmup-logits-go-NaN bug surfaced when `--mmproj` is supplied:
/// running the chat warmup BEFORE any other Metal device activity (mmproj
/// load, embedding-model load, ViT warmup) keeps the chat-model GPU
/// state stable.  The hot-swap orchestrator preserves that ordering by
/// running each engine's warmup synchronously inside `load_engine` so
/// the engine returned to the caller is fully primed.
///
/// **Errors** propagate from header parse, weights load, tokenizer parse,
/// chat-template resolution, or warmup — every failure is fatal to the
/// load attempt.  Caller decides how to surface it (cmd_serve aborts the
/// boot; HotSwapManager will return the error to the request handler).
/// Derive a stable pool key for a filesystem-path passthrough — used
/// when `auto_pipeline::resolve_or_prepare_model` returns
/// `repo_id: None` (the operator passed `--model /path/to.gguf`
/// instead of an HF repo-id).  The file stem matches what the engine
/// itself uses for `model_id()` when GGUF metadata lacks `general.name`,
/// so the pool key matches the surface every other code path sees.
///
/// Deterministic per identical input: two requests resolving to the
/// same on-disk file yield the same key, the pool reuses the engine.
pub fn pool_key_for_path(path: &Path) -> String {
    path.file_stem()
        .map(|s| s.to_string_lossy().into_owned())
        .unwrap_or_else(|| path.to_string_lossy().into_owned())
}

pub fn load_engine(path: &Path, config: &multi_model::EngineConfig) -> Result<api::engine::Engine> {
    anyhow::ensure!(path.exists(), "Model not found: {}", path.display());
    // Header-only parse surfaces bad magic + populates the diagnostic
    // log line.  Cheap (memory-mapped header, no tensor read).
    //
    // ADR-005 Phase 4 reopen iter-215 Wedge-2: the pre-iter-215
    // arch-detect bail for qwen35 / qwen35moe (commit 064797e) is
    // replaced by actual SERVE-side dispatch.  `LoadedModel::load`
    // (api/engine.rs) reads `general.architecture` and routes to
    // `Qwen35LoadedModel::load` for Qwen3.5/3.6 or
    // `GemmaLoadedModel::load` otherwise; the worker thread arm for
    // the Qwen35 variant returns HTTP 501 with an operator-actionable
    // message (Wedge-3 deferred follow-up wires the live forward
    // pass).  See `LoadedModel::load` for the dispatch surface.
    {
        let gguf = mlx_native::gguf::GgufFile::open(path)
            .map_err(|e| anyhow::anyhow!("GGUF header parse failed: {e}"))?;
        let arch = gguf
            .metadata_string("general.architecture")
            .map(|s| s.to_string())
            .unwrap_or_default();
        tracing::info!(
            path = %path.display(),
            tensors = gguf.tensor_count(),
            metadata = gguf.metadata_count(),
            arch = %arch,
            "Validated GGUF header"
        );
    }

    let load_opts = api::engine::LoadOptions {
        model_path: path.to_path_buf(),
        tokenizer_path: config.tokenizer_path.clone(),
        config_path: config.config_path.clone(),
        // ADR-020 AC#5 Iter D — propagated from `cmd_serve`'s
        // `args.dwq_overlay` via `multi_model::EngineConfig`.
        dwq_overlay_path: config.dwq_overlay_path.clone(),
        // ADR-027 Phase A iter-6b.2: cmd_serve honors HF2Q_KV_PERSIST.
        kv_persist_dir: std::env::var("HF2Q_KV_PERSIST")
            .ok()
            .filter(|s| !s.is_empty())
            .map(std::path::PathBuf::from),
    };
    let mut loaded = api::engine::LoadedModel::load(&load_opts)?;
    // ADR-017 Phase E.a iter-2: thread the metrics sink onto the
    // GemmaLoadedModel BEFORE Engine::spawn moves `loaded` into the
    // worker. Qwen35 / Qwen3VlText variants don't yet have an LCP
    // probe site (their worker arm returns 501 before any prefill);
    // skip them. None on the EngineConfig ⇒ no-op (test path).
    //
    // ADR-017 Phase B-hybrid.1: Qwen35 ALSO gains kv_metrics_sink so
    // its probe (added in this iter) can record `lcp_lookups_total`
    // and `lcp_detected_total` on the SHARED `KvSpillCounters` Arc.
    if let Some(sink) = config.kv_metrics_sink.as_ref() {
        match &mut loaded {
            api::engine::LoadedModel::Gemma(g) => {
                g.kv_metrics_sink = Some(std::sync::Arc::clone(sink));
            }
            api::engine::LoadedModel::Qwen35(q) => {
                q.kv_metrics_sink = Some(std::sync::Arc::clone(sink));
            }
            api::engine::LoadedModel::Qwen3VlText(_) => {
                // Qwen3VlText path doesn't have an LCP probe yet.
            }
            api::engine::LoadedModel::Deepseek4(_) => {
                // DeepSeek-V4 accounts exact-prefix hits in generation stats.
            }
        }
    }
    // ADR-040 Phase C iter-4 (C4) — route through `spawn_with_mode` so
    // the operator-chosen `EngineMode` (parsed from `--scheduler` /
    // `HF2Q_SCHEDULER` upstream in `cmd_serve`) reaches the worker
    // thread. `EngineConfig::default()` carries `EngineMode::SerialFifo`,
    // which `spawn_with_mode` delegates byte-for-byte to the legacy
    // 3-arg `Engine::spawn` per iter-1.5 F1 (engine.rs:2723-2748) — so
    // the env-absence path stays byte-equivalent per ADR-040 §3.6.
    //
    // SlotAware is rejected with `EngineSpawnError::ModeNotYetWired`
    // until iter-2b (Qwen35 worker arm) + iter-2c (Gemma 4) land; we
    // surface the typed error as an `anyhow::Error` carrying the
    // ADR-040-prefixed Display string so `cmd_serve` can print it to
    // stderr and exit non-zero (fail-loud per ADR-040 §7 mantra — no
    // silent fallback to SerialFifo).
    let engine = api::engine::Engine::spawn_with_mode(
        loaded,
        config.queue_capacity,
        None,
        config.engine_mode,
    )
    .map_err(|e| {
        anyhow::anyhow!(
            "ADR-040 Phase C iter-4 (C4): Engine::spawn_with_mode rejected the \
             requested EngineMode: {e}. Either select `--scheduler fifo_serial` \
             (the default; ADR-005 byte-equivalent path), or wait for the iter \
             named in the error message to land."
        )
    })?;
    load_info::emit_tracing(engine.info());

    if config.warmup_synchronously {
        // Warm up the engine SYNCHRONOUSLY here, BEFORE any other Metal
        // device activity (mmproj load, embedding-model load, ViT
        // warmup) happens.  This is iter-103's fix for the
        // chat-warmup-logits-go-NaN bug surfaced when `--mmproj` is
        // supplied: bisection showed loading the mmproj weights
        // (~1 GB of fresh F16-→F32 dequant'd Metal buffers) corrupts
        // the chat-model's pre-warmup state somehow (likely Metal-
        // driver buffer interleaving on Apple Silicon unified memory),
        // making every chat-model logit NaN at first decode.  Running
        // the chat warmup BEFORE any other Metal work is a structural
        // fix: the chat-model's GPU state is fully exercised + stable
        // by the time the mmproj load adds buffers.  As a happy side
        // effect, /readyz returns 200 immediately upon serving instead
        // of after the previously-async warmup completed.
        //
        // Uses a temp tokio runtime to drive the async API.  The
        // serve-time runtime is built later and is independent of
        // this one.
        let warmup_rt = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .context("build tokio runtime for synchronous engine warmup")?;
        let warmup_started = std::time::Instant::now();
        warmup_rt
            .block_on(engine.warmup())
            .context("synchronous engine warmup")?;
        tracing::info!(
            elapsed_ms = warmup_started.elapsed().as_millis() as u64,
            "Engine warmed up synchronously (pre-mmproj order — iter-103 fix)"
        );
        drop(warmup_rt);
    }

    Ok(engine)
}

/// Run the `serve` subcommand — start the OpenAI-compatible HTTP API server.
///
/// ADR-005 Phase 2a iter-2 backbone: exposes `/health`, `/readyz`,
/// `/v1/models`, `/v1/models/:id`. Chat completions + embeddings land in
/// the next iter with the engine wiring.
///
/// Behavior:
///   1. Build `ServerConfig` from CLI args + env vars.
///   2. If `--model` is supplied, validate the GGUF header opens cleanly
///      (fail-fast on bad weights, per Decision #15). No tensor data is
///      read — that happens when the engine loads in iter 3.
///   3. Build the axum router and bind the listener.
///   4. Serve until SIGINT / SIGTERM (graceful shutdown per Decision #17).
/// ADR-017 §R-F1 startup-disable predicate: returns `true` iff the
/// kv-persist substrate should be constructed at startup.
///
/// Contract:
///   * `flag` — `--kv-persist=PATH` argument as a borrowed string slice
///     (`None` = flag absent).
///   * `env` — value of the `HF2Q_KV_PERSIST` env var (`None` = unset).
///
/// Rules:
///   * No flag → never enable (matches pre-ADR-017 behavior).
///   * Flag present + env unset / empty / any value other than `"0"`
///     (after trim) → enable.
///   * Flag present + env trims to exactly `"0"` → disable
///     (operator emergency-override per `docs/operating-kv-cache.md` §10).
///
/// Env-driven semantics are process-scoped: operators restart
/// `cmd_serve` to re-enable. The pure-function form keeps the policy
/// trivially unit-testable without spinning up the binary.
pub(crate) fn should_enable_kv_persist(flag: Option<&str>, env: Option<&str>) -> bool {
    flag.is_some() && env.map(|v| v.trim()).unwrap_or("") != "0"
}

pub(crate) fn maybe_print_serve_banner<W: std::io::Write>(
    info: &load_info::LoadInfo,
    w: &mut W,
    tty: bool,
    quiet: bool,
) -> std::io::Result<()> {
    if tty && !quiet {
        load_info::print_banner(info, w, tty)?;
    }
    Ok(())
}

/// ADR-040 Phase C iter-4 (C4) — central authority for resolving the
/// operator's `--scheduler` / `--max-slots` CLI selection against the
/// corresponding `HF2Q_SCHEDULER` / `HF2Q_MAX_SLOTS` env vars into a
/// concrete [`api::engine::EngineMode`].
///
/// # Backward-compat contract (ADR-040 §3.6)
///
/// When BOTH `scheduler` arguments are `None` (CLI flag absent AND env
/// var unset / empty), this function returns
/// [`api::engine::EngineMode::SerialFifo`] — preserving the ADR-005
/// Decision #2 + #19 production path byte-for-byte. The `max_slots`
/// inputs are IGNORED on the SerialFifo path (the legacy
/// 3-arg `Engine::spawn` is single-slot by definition; `max_slots` is
/// a SlotAware-only concept).
///
/// # Precedence
///
/// - `scheduler_cli` (Some) overrides `scheduler_env` (mirrors the
///   `--auth-token` / `HF2Q_AUTH_TOKEN` precedence cmd_serve already
///   uses for other knobs).
/// - `max_slots_cli` (Some) overrides `max_slots_env`.
/// - Whitespace-only env values are treated as unset (mirrors
///   `should_enable_kv_persist`'s `.trim()` discipline).
///
/// # Case-insensitivity (env path)
///
/// The env-var path parses `fifo_serial` / `inflight_batched` /
/// `FIFO_SERIAL` / `INFLIGHT_BATCHED` interchangeably — operators
/// occasionally export env vars in upper-case by convention. The CLI
/// path uses clap's `ValueEnum`, which is case-insensitive by default
/// for snake-case Rust variants.
///
/// # Validation
///
/// Returns `Err(String)` (operator-facing message) on:
/// - Unknown scheduler value at the env-var layer (the CLI is gated by
///   clap so this can only fire via env).
/// - `max_slots == 0` (per ADR-040 iter-2.5 F3a `.max(1)` discipline,
///   refused loudly rather than coerced).
/// - `HF2Q_MAX_SLOTS` env value that doesn't parse as `u32`.
///
/// # Pure function (no env reads / no I/O)
///
/// The caller threads the parsed CLI flag + the env-var value in.
/// This keeps the function unit-testable without `std::env::set_var`
/// races (the existing `should_enable_kv_persist` follows the same
/// pattern for the same reason).
pub(crate) fn parse_scheduler_config(
    scheduler_cli: Option<cli::SchedulerArg>,
    scheduler_env: Option<&str>,
    max_slots_cli: Option<u32>,
    max_slots_env: Option<&str>,
) -> std::result::Result<api::engine::EngineMode, String> {
    use api::engine::EngineMode;

    // ---- 1. Resolve scheduler choice ----
    //
    // CLI flag wins; env is the fallback. Whitespace-only env values
    // are treated as unset (trim semantics match `should_enable_kv_persist`).
    let scheduler_from_env: Option<cli::SchedulerArg> =
        match scheduler_env.map(|s| s.trim()).filter(|s| !s.is_empty()) {
            None => None,
            Some(raw) => {
                let lower = raw.to_ascii_lowercase();
                match lower.as_str() {
                    "fifo_serial" => Some(cli::SchedulerArg::FifoSerial),
                    "inflight_batched" => Some(cli::SchedulerArg::InflightBatched),
                    _ => {
                        return Err(format!(
                            "ADR-040 C4: HF2Q_SCHEDULER={raw:?} is not a recognized \
                         scheduler policy. Supported values (case-insensitive): \
                         `fifo_serial` (default; ADR-005 byte-equivalent path), \
                         `inflight_batched` (ADR-040 slot-aware path, gated on \
                         iter-2b/2c worker-arm landing). Unset the env var to \
                         use the default."
                        ));
                    }
                }
            }
        };
    let scheduler = scheduler_cli.or(scheduler_from_env);

    // ---- 2. Resolve max_slots ----
    //
    // CLI flag wins; env is the fallback. `HF2Q_MAX_SLOTS=0` is
    // refused loudly (per ADR-040 iter-2.5 F3a). Whitespace-only env
    // values are treated as unset.
    let max_slots_from_env: Option<u32> =
        match max_slots_env.map(|s| s.trim()).filter(|s| !s.is_empty()) {
            None => None,
            Some(raw) => match raw.parse::<u32>() {
                Ok(parsed) => Some(parsed),
                Err(err) => {
                    return Err(format!(
                        "ADR-040 C4: HF2Q_MAX_SLOTS={raw:?} does not parse as a \
                     non-negative u32: {err}. Supply a positive integer (default \
                     {}), or unset the env var.",
                        DEFAULT_MAX_SLOTS_UNDER_INFLIGHT
                    ));
                }
            },
        };
    let max_slots_requested = max_slots_cli.or(max_slots_from_env);
    if let Some(0) = max_slots_requested {
        return Err(format!(
            "ADR-040 C4: --max-slots=0 / HF2Q_MAX_SLOTS=0 is rejected per \
             ADR-040 iter-2.5 F3a (FifoSchedulerAdapter `.max(1)` discipline \
             applied at the CLI layer instead of silently coerced). Either \
             omit the flag (defaults to {} when `--scheduler inflight_batched` \
             is selected; ignored otherwise), or supply a positive integer.",
            DEFAULT_MAX_SLOTS_UNDER_INFLIGHT
        ));
    }

    // ---- 3. Build the EngineMode ----
    match scheduler {
        // ADR-040 §3.6 backward-compat — env-absence is byte-equivalent
        // to pre-ADR-040. `max_slots` (if any) is IGNORED on the
        // SerialFifo path; the legacy 3-arg `Engine::spawn` is
        // single-slot by definition.
        None | Some(cli::SchedulerArg::FifoSerial) => Ok(EngineMode::SerialFifo),
        Some(cli::SchedulerArg::InflightBatched) => {
            let max_slots = max_slots_requested.unwrap_or(DEFAULT_MAX_SLOTS_UNDER_INFLIGHT);
            // Defensive: parsed-zero already rejected above. The
            // `.max(1)` here is a defense-in-depth pin for any future
            // refactor that loses the zero-check above.
            Ok(EngineMode::SlotAware {
                max_slots: max_slots.max(1),
            })
        }
    }
}

/// ADR-040 §3.4 — default `max_slots` under `EngineMode::SlotAware`.
/// Half the reopen-trigger threshold of `8`, so the first ramp deploys
/// with headroom.
pub(crate) const DEFAULT_MAX_SLOTS_UNDER_INFLIGHT: u32 = 4;

pub fn cmd_serve(args: cli::ServeArgs) -> Result<()> {
    use api::schema::OverflowPolicy;
    use api::state::ServerConfig;

    // --- Resolve config ---
    let auth_token = args.auth_token.clone().or_else(|| {
        std::env::var("HF2Q_AUTH_TOKEN")
            .ok()
            .filter(|s| !s.is_empty())
    });

    let overflow_policy = match args.overflow_policy {
        cli::OverflowPolicyArg::Reject => OverflowPolicy::Reject,
        cli::OverflowPolicyArg::TruncateLeft => OverflowPolicy::TruncateLeft,
        cli::OverflowPolicyArg::Summarize => OverflowPolicy::Summarize,
    };

    let cache_dir = args
        .cache_dir
        .clone()
        .or_else(api::state::default_cache_dir);

    // ADR-040 Phase C iter-4 (C4) — resolve `--scheduler` + `--max-slots`
    // against `HF2Q_SCHEDULER` + `HF2Q_MAX_SLOTS` into a concrete
    // `EngineMode`. The pure-function parser at
    // `parse_scheduler_config` is unit-tested without env mutation; this
    // call site is the only `std::env::var` lookup for these two knobs.
    //
    // Per ADR-040 §3.6 backward-compat: with the flag absent AND the
    // env unset/empty, this returns `EngineMode::SerialFifo` —
    // byte-equivalent to pre-ADR-040. On a parse failure (bogus
    // scheduler name, `max_slots=0`, non-u32 env), we propagate the
    // operator-facing message via `anyhow!` so `cmd_serve` exits
    // non-zero before binding the listener (fail-loud per §7 mantra).
    let scheduler_env = std::env::var("HF2Q_SCHEDULER").ok();
    let max_slots_env = std::env::var("HF2Q_MAX_SLOTS").ok();
    let engine_mode = parse_scheduler_config(
        args.scheduler,
        scheduler_env.as_deref(),
        args.max_slots,
        max_slots_env.as_deref(),
    )
    .map_err(|msg| anyhow::anyhow!("{msg}"))?;
    tracing::info!(
        engine_mode = ?engine_mode,
        scheduler_cli = ?args.scheduler,
        scheduler_env = ?scheduler_env,
        max_slots_cli = ?args.max_slots,
        max_slots_env = ?max_slots_env,
        "ADR-040 C4: resolved scheduler policy"
    );

    let config = ServerConfig {
        host: args.host.clone(),
        port: args.port,
        auth_token,
        cors_allowed_origins: args.cors_origins.clone(),
        queue_capacity: args.queue_capacity,
        max_concurrent_requests: 0,
        request_timeout_seconds: 0,
        default_overflow_policy: overflow_policy,
        cache_dir,
        system_fingerprint: Some(system_fingerprint()),
    };

    // Warn when exposing beyond localhost. Decision #7 + #13 — public-internet
    // is NOT a supported deployment target.
    if args.host == "0.0.0.0" {
        tracing::warn!(
            "Server bound to 0.0.0.0 — exposes API on all interfaces. \
             Public-internet exposure is NOT a supported deployment target \
             (see ADR-005 Decision #13: reverse-proxy assumption). \
             For LAN-only Open WebUI, this is the intended usage."
        );
    }

    // --- AppState construction (iter-209) ---
    // Build the pool-backed AppState before any model resolution: opens
    // the on-disk cache once + detects hardware once + constructs an
    // empty `HotSwapManager<Engine>` sized off the unified-memory budget
    // per ADR-005 line 929 (80% default).  The same `cache` + `hardware`
    // are shared with request-time auto_pipeline resolution.
    let default_model_arg = args
        .model
        .as_ref()
        .map(|p| p.to_string_lossy().into_owned());
    let mut state = api::AppState::new_for_serve(
        config.clone(),
        args.no_integrity,
        config.queue_capacity,
        default_model_arg.clone(),
    )?;

    // --- ADR-017 Phase C.1 — optional persistent block-prefix KV cache ---
    // When `--kv-persist=PATH` is set, replace the AppState's
    // NoopKvSpiller-backed HotSwapManager with one wired to the
    // Phase A-substrate (DiskBlockStore + AsyncWriterHandle +
    // BlockPrefixCacheSpiller) and a per-loaded-family
    // EngineBindable registry. Off-path (flag absent) is byte-
    // identical to pre-C.1 — the existing AppState manager stays
    // wired with NoopKvSpiller.
    //
    // The substrate variables outlive the cmd_serve scope via Arc
    // captures inside the new HotSwapManager. The recovery scan
    // runs synchronously here so a previously-written cache becomes
    // available before the optional pre-warm load_or_get fires.
    //
    // ADR-017 §R-F1 startup-disable override: `HF2Q_KV_PERSIST=0`
    // (exact match, after trim) overrides `--kv-persist=PATH` and
    // leaves the NoopKvSpiller-backed AppState manager wired. This
    // closes the operator-runbook §10 gap (`docs/operating-kv-cache.md`)
    // — operators emergency-disable by setting the env var and
    // restarting (env vars are process-scoped, so true mid-flight
    // disable is restart-driven; that's the standard operator
    // pattern for env-driven config). Restart without
    // `HF2Q_KV_PERSIST=0` to re-enable. The decision predicate is
    // factored into `should_enable_kv_persist` below for unit
    // testing.
    let kv_persist_env = std::env::var("HF2Q_KV_PERSIST").ok();
    let kv_persist_flag_path = args.kv_persist_path.as_ref();
    let kv_persist_enabled = should_enable_kv_persist(
        kv_persist_flag_path.map(|p| p.to_string_lossy()).as_deref(),
        kv_persist_env.as_deref(),
    );
    if !kv_persist_enabled && kv_persist_flag_path.is_some() {
        // Flag was supplied but env override forced disable — surface
        // a single warn line so operators see the override took
        // effect without grepping logs for absent counters.
        let path_for_log = kv_persist_flag_path
            .map(|p| p.display().to_string())
            .unwrap_or_default();
        tracing::warn!(
            kv_persist_path = %path_for_log,
            "ADR-017 R-F1 override: HF2Q_KV_PERSIST=0 — disabling kv-persist \
             despite --kv-persist={path_for_log}. Operator emergency-disable per \
             operating-kv-cache.md §10. Restart without HF2Q_KV_PERSIST=0 to \
             re-enable.",
            path_for_log = path_for_log,
        );
    }
    let kv_persist_loader_wrapper: Option<
        std::sync::Arc<crate::serve::kv_persist::LoaderWrapper<api::engine::Engine>>,
    > = if let Some(cache_dir) = args.kv_persist_path.as_ref().filter(|_| kv_persist_enabled) {
        use crate::serve::kv_persist::families::gemma4_dense::{
            Gemma4DenseConfig, Gemma4DenseSpillFactory,
        };
        use crate::serve::kv_persist::families::tq_packed::{
            flags as tq_flags, TqBitsPerCoord, TqPackedConfig, TqPackedSpillFactory,
        };
        use crate::serve::kv_persist::registry::FamilyHookFactory;
        use crate::serve::kv_persist::{
            AsyncWriterHandle, BlockPrefixCacheSpiller, DiskBlockStore, KvPersistRegistry,
            LoaderWrapper, StubGemma4Spill, DEFAULT_CHANNEL_CAPACITY,
        };
        use crate::serve::multi_model::{DefaultModelLoader, HotSwapManager, LoadedPool};
        use std::path::PathBuf;
        use std::sync::{Arc, Mutex};

        // 1. Ensure cache_dir exists.
        std::fs::create_dir_all(cache_dir).with_context(|| {
            format!(
                "ADR-017 C.1: create kv-persist cache dir at {}",
                cache_dir.display()
            )
        })?;

        // 2. Recovery scan — rebuild BlockIndex from on-disk
        //    envelopes left by a prior run. Returns
        //    `(BlockIndex, RecoveryReport)` per recovery.rs:110.
        //
        //    ADR-017 §R-F7: route through `recover_from_disk_with_counters`
        //    so the scan-time quarantine moves bump
        //    `hf2q_kv_quarantined_total{reason=...}` against the same
        //    AppState-owned counters Arc the /metrics handler reads.
        //    Upcast `Arc<KvSpillCounters>` → `Arc<dyn KvCacheMetricsSink>`
        //    here because the substrate's seam is trait-objected (per
        //    `kv_persist::metrics` module docs — keeps the lib facade
        //    decoupled from `serve::api`).
        let metrics_sink: Arc<dyn crate::serve::kv_persist::metrics::KvCacheMetricsSink> =
            Arc::clone(&state.kv_spill_counters)
                as Arc<dyn crate::serve::kv_persist::metrics::KvCacheMetricsSink>;
        let (recovered_index, recovery_report) =
            crate::serve::kv_persist::recover_from_disk_with_counters(
                cache_dir,
                Some(&metrics_sink),
            )
            .with_context(|| format!("ADR-017 C.1: recover_from_disk({})", cache_dir.display()))?;
        tracing::info!(
            cache_dir = %cache_dir.display(),
            blocks_indexed = recovery_report.blocks_indexed,
            blocks_quarantined = recovery_report.blocks_quarantined,
            bytes_indexed = recovery_report.bytes_indexed,
            elapsed_ms = recovery_report.elapsed_ms,
            "ADR-017 C.1: kv-persist recovery scan complete"
        );

        // 3. DiskBlockStore — owns the read-path file I/O.
        //
        //    ADR-017 P1-3 (adversarial review fix): wire the on-disk
        //    LRU budget from `HF2Q_KV_PERSIST_BUDGET_BYTES` (u64,
        //    bytes). Default `0` = unlimited (matches the pre-P1-3
        //    behavior so the env var is purely additive). Per
        //    ADR-017 §R7 the intended future default is "10% of
        //    unified RAM" (~12.8 GiB on a 128-GiB M5 Max); that
        //    requires a `mlx_native::sysinfo::physical_ram_bytes()`
        //    helper which does not yet exist, and per the standing
        //    directive we do NOT add a new mlx-native dep just for
        //    this. Operators set the env var explicitly until §R7's
        //    helper lands.
        //
        //    Construct via `new_with_index` (kept at 0 for
        //    source-compat with existing callers), then override
        //    through `set_budget_bytes` so the AtomicU64 holds the
        //    parsed value before any eviction call fires.
        let kv_persist_budget_bytes: u64 = match std::env::var("HF2Q_KV_PERSIST_BUDGET_BYTES") {
            Ok(raw) => match raw.trim().parse::<u64>() {
                Ok(parsed) => parsed,
                Err(err) => {
                    tracing::warn!(
                        raw = %raw,
                        error = %err,
                        "ADR-017 P1-3: HF2Q_KV_PERSIST_BUDGET_BYTES \
                         parse failed; defaulting to 0 (unlimited)"
                    );
                    0
                }
            },
            Err(_) => 0,
        };
        let store = Arc::new(
            DiskBlockStore::new_with_index(cache_dir.clone(), recovered_index, 0).with_context(
                || {
                    format!(
                        "ADR-017 C.1: DiskBlockStore::new_with_index({})",
                        cache_dir.display()
                    )
                },
            )?,
        );
        store.set_budget_bytes(kv_persist_budget_bytes);
        // ADR-017 §R-F7: wire the DiskBlockStore to the AppState's
        // kv_spill_counters so eviction bumps
        // `hf2q_kv_cache_evictions_total{trigger="budget_overflow"}`
        // against the same Arc the /metrics handler reads. AppState
        // also gets a clone of the store handle so the gauges
        // `hf2q_kv_cache_bytes_on_disk` + `hf2q_kv_cache_blocks_total`
        // resolve at scrape time against the live BlockIndex.
        store.set_kv_counters(Arc::clone(&metrics_sink));
        state.kv_disk_store = Some(Arc::clone(&store));
        tracing::info!(
            budget_bytes = kv_persist_budget_bytes,
            "ADR-017 P1-3: HF2Q_KV_PERSIST_BUDGET_BYTES wired (0 = unlimited)"
        );

        // 4. AsyncWriterHandle — owns the write-path background
        //    worker. DEFAULT_CHANNEL_CAPACITY is the iter-212
        //    contract default; a future Phase D iter exposes it via
        //    a CLI knob.
        let writer = Arc::new(AsyncWriterHandle::spawn(
            Arc::clone(&store),
            DEFAULT_CHANNEL_CAPACITY,
        ));

        // 5. BlockPrefixCacheSpiller — owns the lifecycle. Per-
        //    family hooks register below.
        let spiller: Arc<BlockPrefixCacheSpiller<api::engine::Engine>> = Arc::new(
            BlockPrefixCacheSpiller::new(Arc::clone(&store), Arc::clone(&writer)),
        );

        // 6. KvPersistRegistry — the Phase C.1 EngineBindable
        //    registry. Populated once per known family below.
        let registry = Arc::new(KvPersistRegistry::new());

        // ADR-017 Closure iter-10 (2026-05-05): wire the registry into
        // the spiller so `BlockPrefixCacheSpiller::drop_family` also
        // unregisters the registry-side `Arc<dyn EngineBindable>` on
        // permanent eviction. Without this, the registry's `hooks`
        // map retains an `Arc<Gemma4DenseSpill>` whose `engine_arc`
        // field holds an `Engine` clone (Arc<EngineInner>) — keeping
        // the worker thread + LoadedModel weights alive forever
        // (~15 GB anon memory leak per stress iter, verified iter-10c).
        spiller.set_registry(Arc::clone(&registry));

        // 7. Stub-family registration: register `StubGemma4Spill`
        //    for the operator-supplied --model (if any). The stub
        //    returns Skipped on every snapshot/restore call (per
        //    spiller.rs:506-528) and a no-op on bind/unbind (per
        //    the spiller.rs EngineBindable impl), so the on-path
        //    is observable but functionally inert.
        //
        //    B-dense.2 swaps `StubGemma4Spill` for the real
        //    `Gemma4DenseSpill` once the GGUF metadata extraction
        //    + EngineHandle construction lands.
        if let Some(model_arg) = default_model_arg.as_deref() {
            let stub = Arc::new(StubGemma4Spill);
            let stub_for_spiller: Arc<Mutex<dyn crate::serve::kv_persist::KvCacheSpill>> =
                Arc::new(Mutex::new(StubGemma4Spill));
            let stub_for_registry: Arc<dyn crate::serve::kv_persist::EngineBindable> = stub.clone();
            // Keys: derive a synthetic (repo, quant) the same way
            // the pre-warm path will (so the spiller's lookup hits
            // when load_or_get fires). Phase D's GGUF-derived
            // (repo, quant) lands later.
            let pool_repo = pool_key_for_path(&PathBuf::from(model_arg));
            let pool_quant = quant_select::QuantType::Q4_K_M;
            spiller.register_family(pool_repo.clone(), pool_quant, stub_for_spiller);
            registry.register(pool_repo.clone(), pool_quant, stub_for_registry);
            tracing::info!(
                repo = %pool_repo,
                quant = %pool_quant.as_str(),
                "ADR-017 C.1: registered StubGemma4Spill for operator --model"
            );

            // ADR-017 B-dense.2 — register a Gemma4DenseSpillFactory
            // alongside the C.1 stub. On the first successful engine
            // load delivering an `Arc<EngineHandle>` (the production
            // path drives this from a separate post-load bind in
            // cmd_serve, not the auto-LoaderWrapper-bind path), the
            // registry's `try_substitute_on_load` will materialize a
            // real Gemma4DenseSpill and atomically substitute BOTH
            // the spiller's family hook AND the registry's bindable
            // hook. Until that fires, the C.1 stub remains the
            // active hook (Skipped on every snapshot/restore).
            //
            // The shape config wired here is a *fallback* used only
            // if the descriptor-based production substitution path
            // fails to fire. Real GGUF-derived shape extraction is
            // already in production at
            // `src/serve/api/engine.rs:2063-2112`
            // (`KvSpillDescriptor::from_gemma_loaded_model`) — it
            // captures `LayerSpec` / `DenseKvBuffers` /
            // `sliding_window` from the loaded `MlxModelWeights` at
            // engine spawn. The factory consumes that descriptor at
            // `src/serve/kv_persist/families/gemma4_dense.rs:1464-1479`
            // (the `Arc<Engine>` downcast branch overrides the
            // shape-equivalent fields of this fallback cfg with
            // descriptor-derived values, falling back to this
            // cfg's `max_decode_tokens` only when the descriptor
            // value is 0). The factory itself is fully
            // shape-parametric.
            //
            // The matrix harness in `tests/kv_persist_gemma4_roundtrip.rs`
            // exercises the seam end-to-end via subprocess + real
            // GGUF.
            let fallback_cfg = Gemma4DenseConfig {
                // Two layers — one Sliding, one Full — minimum that
                // exercises both layer-type branches in the spill's
                // capacity_for_layer / alloc_layer paths. The real
                // Gemma 4 26B has 64 layers (48 sliding + 16
                // full-attention); these fallback values are
                // *correct shape* but not *correct length*. They
                // only take effect if the production
                // descriptor-downcast path at
                // gemma4_dense.rs:1464 does NOT fire (e.g. the
                // engine is wrapped in an Arc<EngineHandle> and
                // hits the B-dense.1 backwards-compat fallback at
                // gemma4_dense.rs:1485).
                layer_types: vec![
                    crate::serve::config::LayerType::Sliding,
                    crate::serve::config::LayerType::Full,
                ],
                nkv_heads: vec![8, 2],
                head_dim: vec![256, 512],
                kv_dtype: mlx_native::DType::F32,
                sliding_window: 4096,
                max_decode_tokens: 8192,
            };
            // **ADR-017 B-tq.4 iter-2 — single-registration policy.**
            //
            // Decision: register EITHER Gemma4DenseSpillFactory OR
            // TqPackedSpillFactory per (repo, quant) FamilyKey, NOT
            // both.  Reason: `KvPersistRegistry::register_factory`
            // uses `HashMap<FamilyKey, Arc<dyn FamilyHookFactory>>`
            // (registry.rs:158); the second registration overwrites
            // the first.  Dual-registration would require either
            // (a) a `Vec<Factory>` per key (changes registry trait
            // surface) or (b) two distinct FamilyKeys (changes the
            // bind/unbind discovery path).  Neither is justified
            // when the runtime is in exactly one mode at a time —
            // `HF2Q_TQ_KV=1` puts the engine on the TQ-active path
            // for every layer and request; the dense factory's
            // descriptor closure already returns None when
            // `dense_kvs` is None.  So at runtime exactly one is
            // active; we register exactly one.
            //
            // Detection: `is_tq_active_mode()` reads `HF2Q_TQ_KV`
            // env at cmd_serve startup.  Mid-process toggling of
            // the env doesn't re-shape the descriptor (matches the
            // Gemma4Dense pattern at engine.rs:2149-2158 — env read
            // ONCE at spawn).  This iter is correct for the static
            // operator-time mode selection; future ADR-017 B-tq.5
            // (if needed) can lift the FamilyKey shape to allow
            // dual-mode swap-on-evict.
            let factory: Arc<dyn FamilyHookFactory> =
                if crate::serve::api::tq_packed_descriptor::is_tq_active_mode() {
                    // **TQ-active path**: register TqPackedSpillFactory.
                    //
                    // Fallback config matches the canonical Gemma 4
                    // 26B layer split (48 sliding + 16 full-attention
                    // = 64 layers) at production shape.  The factory's
                    // descriptor downcast at
                    // `tq_packed.rs::TqPackedSpillFactory::try_construct`
                    // re-derives shape from the live engine's
                    // `MlxModelWeights`; this fallback only takes effect
                    // if the descriptor downcast fails (e.g. an
                    // EngineHandle path materializes — out of scope at
                    // this iter, the auto-LoaderWrapper-bind always
                    // delivers Arc<Engine>).
                    let bits = TqBitsPerCoord::new(
                        crate::serve::api::tq_packed_descriptor::parse_tq_codebook_bits(
                            std::env::var("HF2Q_TQ_CODEBOOK_BITS").ok().as_deref(),
                        ),
                    )
                    .expect("HF2Q_TQ_CODEBOOK_BITS validated to be in {2,3,4,5,6,8}");
                    let tq_fallback_cfg = TqPackedConfig {
                        // Two layers minimum exercising the
                        // sliding/global split (mirrors the dense
                        // fallback at the same indentation level
                        // above).  Real shape is descriptor-driven
                        // post-substitute.
                        num_layers: 2,
                        nkv_heads: vec![8, 2],
                        head_dim: vec![256, 512],
                        bits_per_coord: bits,
                        scale: 1.0,
                        flags: tq_flags::HADAMARD_ROTATED,
                        block_tokens: crate::serve::kv_persist::format::BLOCK_TOKENS,
                    };
                    Arc::new(TqPackedSpillFactory::new(tq_fallback_cfg))
                } else {
                    // **Dense path** (default): register
                    // Gemma4DenseSpillFactory.
                    Arc::new(Gemma4DenseSpillFactory::new(fallback_cfg))
                };
            let factory_kind = if crate::serve::api::tq_packed_descriptor::is_tq_active_mode() {
                "TqPackedSpillFactory"
            } else {
                "Gemma4DenseSpillFactory"
            };
            registry.register_factory(pool_repo.clone(), pool_quant, factory);
            tracing::info!(
                repo = %pool_repo,
                quant = %pool_quant.as_str(),
                factory = factory_kind,
                "ADR-017 B-dense.2 + B-tq.4 iter-2: registered single-mode factory \
                 (lazy real-hook construction at first engine load); HF2Q_TQ_KV \
                 selects TQ-active mode at startup"
            );
        }

        // 8. LoaderWrapper — decorates DefaultModelLoader. The
        //    wrapper's pending_bind slot is armed below, immediately
        //    before the pre-warm load_or_get call (per the
        //    synchronous-contract rationale in
        //    loader_wrapper.rs's module docs).
        let real_loader: Arc<dyn crate::serve::multi_model::ModelLoader<api::engine::Engine>> =
            Arc::new(DefaultModelLoader);
        let wrapper = Arc::new(LoaderWrapper::new(real_loader, Arc::clone(&registry)));
        // ADR-017 B-dense.2 — wire the spiller into the wrapper so a
        // successful factory substitution updates both the registry
        // (handled inside try_substitute_on_load) AND the spiller's
        // register_family table (handled in
        // LoaderWrapper::update_spiller_registration). Order
        // matters: set_spiller MUST be called before any load_or_get
        // arms a pending_bind.
        wrapper.set_spiller(Arc::clone(&spiller));
        let loader_for_manager: Arc<
            dyn crate::serve::multi_model::ModelLoader<api::engine::Engine>,
        > = wrapper.clone();

        // 9. Build a fresh HotSwapManager using new_with_spiller and
        //    swap state.pool to point at it. Re-derive the pool +
        //    counters the same way AppState::new_for_serve did so
        //    behavior diverges from the off-path ONLY in the
        //    spiller wiring.
        let pool = LoadedPool::from_hardware(state.hardware.as_ref());
        let mut manager: HotSwapManager<api::engine::Engine> = HotSwapManager::new_with_spiller(
            pool,
            loader_for_manager,
            Arc::clone(&spiller)
                as Arc<dyn crate::serve::multi_model::KvSpiller<api::engine::Engine>>,
        );
        manager.set_kv_counters(Arc::clone(&state.kv_spill_counters));
        state.pool = Arc::new(std::sync::RwLock::new(manager));
        // ADR-017 Closure iter-2 (2026-05-04): expose the concrete
        // spiller to the graceful-shutdown drain helper. The Arc is
        // also held inside HotSwapManager (as a trait object) so the
        // eviction trigger sites at multi_model.rs:1090,1189 keep
        // working unchanged. Holding both references is intentional —
        // shutdown drain needs concrete-type methods
        // (`pending_writer_queue_depth`) the trait does not expose.
        state.kv_spiller = Some(spiller);

        tracing::info!(
            cache_dir = %cache_dir.display(),
            "ADR-017 C.1: kv-persist spiller substrate wired into HotSwapManager + AppState"
        );

        Some(wrapper)
    } else {
        None
    };

    // --- Optional pre-warm (--model supplied) ---
    // ADR-005 Phase 4 iter-208 (W76): the load path flows through the
    // shared `load_engine` callable via `HotSwapManager::load_or_get`,
    // which dispatches against `DefaultModelLoader`.  iter-209 unifies
    // the startup path with the request-time auto-swap path: both call
    // `pool.load_or_get(...)` against the same manager.  Pre-warming at
    // startup keeps Decision #15 (fail-fast on bad weights) intact and
    // guarantees /readyz returns 200 with a usable pooled engine.
    //
    // Filesystem-path passthrough uses the file stem as the pool's
    // `repo` key + a synthetic `Q4_K_M` quant (the on-disk quant is
    // baked into the file; the pool key just needs determinism per
    // identical input).
    let mut startup_engine_for_banner: Option<api::engine::Engine> = None;
    if let Some(model_arg) = default_model_arg.as_ref() {
        let mut cache_guard = state
            .cache
            .lock()
            .map_err(|e| anyhow::anyhow!("cache mutex poisoned at startup: {e}"))?;
        let resolved = auto_pipeline::resolve_or_prepare_model(
            model_arg,
            &mut cache_guard,
            state.hardware.as_ref(),
            state.no_integrity,
        )
        .context("auto-pipeline: resolve --model into a GGUF path")?;
        // Drop the cache lock before the long-running engine load so
        // request-time resolutions can proceed concurrently if they
        // somehow arrive (test paths) — production startup is
        // single-threaded but the lock-discipline is right anyway.
        drop(cache_guard);

        if let Some(repo) = resolved.repo_id.as_deref() {
            let quant_str: &str = resolved
                .quant
                .map(quant_select::QuantType::as_str)
                .unwrap_or("");
            tracing::info!(
                repo,
                quant = quant_str,
                from_cache = resolved.from_cache,
                gguf = %resolved.gguf_path.display(),
                "auto-pipeline: --model resolved"
            );
        }

        let pool_repo = resolved
            .repo_id
            .clone()
            .unwrap_or_else(|| pool_key_for_path(&resolved.gguf_path));
        let pool_quant = resolved.quant.unwrap_or(quant_select::QuantType::Q4_K_M);
        let engine_config = multi_model::EngineConfig {
            tokenizer_path: args.tokenizer.clone(),
            config_path: args.config.clone(),
            queue_capacity: config.queue_capacity,
            warmup_synchronously: true,
            // ADR-017 Phase E.a iter-2: cmd_serve startup pre-warm
            // path. Thread the same AppState counters Arc here so the
            // initial pre-warmed engine has the same LCP-probe sink
            // wiring as engines admitted later via the auto-pipeline.
            kv_metrics_sink: Some(std::sync::Arc::clone(&state.kv_spill_counters)
                as std::sync::Arc<dyn crate::serve::kv_persist::metrics::KvCacheMetricsSink>),
            // ADR-020 AC#5 Iter D — propagate `--dwq-overlay` flag.
            dwq_overlay_path: args.dwq_overlay.clone(),
            // ADR-040 Phase C iter-4 (C4) — propagate the parsed
            // `EngineMode` so `load_engine` calls
            // `Engine::spawn_with_mode` with the operator's chosen
            // scheduler policy. Default `SerialFifo` keeps the
            // ADR-005 byte-equivalence pledge when no flag/env is set.
            engine_mode,
        };
        // ADR-017 C.1: arm the LoaderWrapper's pending_bind slot for
        // the about-to-fire load_or_get. Synchronous contract — see
        // loader_wrapper.rs's module docs.
        if let Some(wrapper) = kv_persist_loader_wrapper.as_ref() {
            wrapper.set_pending_bind(pool_repo.clone(), pool_quant);
        }

        let mut pool_guard = state
            .pool
            .write()
            .map_err(|e| anyhow::anyhow!("pool rwlock poisoned at startup: {e}"))?;
        let loaded_engine = pool_guard
            .load_or_get(&pool_repo, pool_quant, &resolved.gguf_path, &engine_config)
            .map_err(|e| anyhow::anyhow!("startup pre-warm: {e}"))?;
        startup_engine_for_banner = Some(loaded_engine.engine.clone());
        drop(pool_guard);
        tracing::info!(
            repo = %pool_repo,
            quant = %pool_quant.as_str(),
            "hf2q startup pre-warm: model admitted to pool"
        );
    }

    // --- Optionally validate + load the BERT embedding model config ---
    // Decision: load config only (header parse), NOT weights. Per
    // ADR-005 Phase 2b iter 16: the forward pass that consumes weights
    // lands when live-model validation is possible (OOM-blocked today).
    // The startup failure path still works: a bad GGUF at this path fails
    // the server boot cleanly.
    let embedding_model = if let Some(emb_path) = args.embedding_model.as_ref() {
        anyhow::ensure!(
            emb_path.exists(),
            "Embedding model not found: {}",
            emb_path.display()
        );
        let gguf = mlx_native::gguf::GgufFile::open(emb_path)
            .map_err(|e| anyhow::anyhow!("Embedding GGUF header parse failed: {e}"))?;

        // Sniff the architecture so we dispatch the correct loader +
        // forward path. Per ADR-005 Phase 2b: bge/mxbai are arch="bert"
        // (separate Q/K/V, position_embd, GeLU MLP, optionally CLS pool);
        // nomic-embed-text-v1.5 is arch="nomic-bert" (fused QKV, RoPE,
        // SwiGLU, Mean pool — see `inference::models::nomic_bert`).
        let arch_str = gguf
            .metadata_string("general.architecture")
            .ok_or_else(|| anyhow::anyhow!("Embedding GGUF missing general.architecture"))?
            .to_string();

        // Vocab + tokenizer are shared across the BERT family — both
        // archs serialize their WPM vocab the same way per llama.cpp's
        // `llm_tokenizer_wpm_session::tokenize`. Iter-79 cross-lane
        // edit added bos→cls / eos→sep fallbacks so nomic GGUFs parse
        // through the BertVocab path unchanged.
        let vocab = crate::inference::models::bert::BertVocab::from_gguf(&gguf)
            .map_err(|e| anyhow::anyhow!("Embedding GGUF vocab parse failed: {e}"))?;
        let tokenizer = crate::inference::models::bert::BertWpmTokenizer::new(&vocab);
        let model_id = emb_path
            .file_stem()
            .map(|s| s.to_string_lossy().into_owned())
            .unwrap_or_else(|| "embedding-model".into());

        let device = mlx_native::MlxDevice::new()
            .map_err(|e| anyhow::anyhow!("create MlxDevice for embedding load: {e}"))?;

        let arch = match arch_str.as_str() {
            "bert" => {
                let cfg = crate::inference::models::bert::BertConfig::from_gguf(&gguf)
                    .map_err(|e| anyhow::anyhow!("BERT GGUF config parse failed: {e}"))?;
                crate::inference::models::bert::weights::validate_tensor_set(&gguf, &cfg)
                    .map_err(|e| anyhow::anyhow!("BERT GGUF tensor validation: {e}"))?;
                let weights = crate::inference::models::bert::weights::LoadedBertWeights::load(
                    &gguf, &cfg, device,
                )
                .map_err(|e| anyhow::anyhow!("BERT weights load failed: {e}"))?;
                tracing::info!(
                    path = %emb_path.display(),
                    arch = "bert",
                    hidden = cfg.hidden_size,
                    layers = cfg.num_hidden_layers,
                    pooling = ?cfg.pooling_type,
                    vocab_size = vocab.len(),
                    tensor_count = weights.len(),
                    "Validated embedding GGUF + loaded weights onto device"
                );
                api::state::EmbeddingArch::Bert {
                    config: cfg,
                    weights: std::sync::Arc::new(weights),
                }
            }
            "nomic-bert" => {
                let cfg =
                    crate::inference::models::nomic_bert::NomicBertConfig::from_gguf(&gguf)
                        .map_err(|e| anyhow::anyhow!("nomic-bert GGUF config parse failed: {e}"))?;
                crate::inference::models::nomic_bert::validate_tensor_set(&gguf, &cfg)
                    .map_err(|e| anyhow::anyhow!("nomic-bert GGUF tensor validation: {e}"))?;
                let weights = crate::inference::models::nomic_bert::LoadedNomicBertWeights::load(
                    &gguf, &cfg, device,
                )
                .map_err(|e| anyhow::anyhow!("nomic-bert weights load failed: {e}"))?;
                tracing::info!(
                    path = %emb_path.display(),
                    arch = "nomic-bert",
                    hidden = cfg.hidden_size,
                    layers = cfg.num_hidden_layers,
                    pooling = ?cfg.pooling_type,
                    rope_freq_base = cfg.rope_freq_base,
                    vocab_size = vocab.len(),
                    tensor_count = weights.len(),
                    "Validated embedding GGUF + loaded weights onto device"
                );
                api::state::EmbeddingArch::NomicBert {
                    config: cfg,
                    weights: std::sync::Arc::new(weights),
                }
            }
            other => {
                anyhow::bail!(
                    "embedding GGUF general.architecture='{other}' is not supported. \
                     Phase 2b day-one models: 'bert' (bge / mxbai) and 'nomic-bert' \
                     (nomic-embed-text-v1.5). File: {}",
                    emb_path.display()
                );
            }
        };

        Some(api::state::EmbeddingModel {
            gguf_path: emb_path.clone(),
            vocab: std::sync::Arc::new(vocab),
            tokenizer: std::sync::Arc::new(tokenizer),
            model_id,
            arch: Some(arch),
        })
    } else {
        None
    };

    // --- Optionally validate + load the mmproj (multimodal projector) ---
    // Header parse only; weight loading lands alongside the ViT forward
    // pass (ADR-005 Phase 2c Task #15). Fail fast if the file is absent
    // or malformed so the server never advertises multimodal capability
    // it can't back.
    let mmproj = if let Some(mmp_path) = args.mmproj.as_ref() {
        anyhow::ensure!(
            mmp_path.exists(),
            "mmproj not found: {}",
            mmp_path.display()
        );
        let gguf = mlx_native::gguf::GgufFile::open(mmp_path)
            .map_err(|e| anyhow::anyhow!("mmproj GGUF header parse failed: {e}"))?;
        let mmp_config = crate::inference::vision::mmproj::MmprojConfig::from_gguf(&gguf)
            .map_err(|e| anyhow::anyhow!("mmproj GGUF config parse failed: {e}"))?;
        // Walk the GGUF's tensor list against the arch-agnostic
        // required set (iter 30 + iter 31). Fails fast on an incomplete
        // producer rather than hitting NotFound mid-forward-pass.
        let actual_names: Vec<&str> = gguf.tensor_names();
        crate::inference::vision::mmproj::validate_tensor_set(&mmp_config, &actual_names)
            .map_err(|e| anyhow::anyhow!("mmproj GGUF tensor-set validation: {e}"))?;
        // Detect the arch profile so forward-pass dispatch knows
        // which per-block-norm shape to expect (Gemma 4 SigLIP vs
        // classic CLIP vs Qwen3-VL SigLIP vs Unknown).
        //
        // 2026-05-02 Wedge-4c.5 Phase-2c (Codex review of 2eb1e36):
        // call the projector-aware detector here, NOT the tensor-only
        // `detect_arch_profile`. Real Qwen3-VL GGUFs may flag DeepStack
        // at indices > 0 (e.g. [3, 7, 15, 23] for a 24-layer ViT),
        // which would leave `v.deepstack.0.fc1.weight` absent and the
        // tensor-only detector would emit `Unknown`, blocking even
        // text-only chat against a perfectly valid Qwen3-VL projector.
        // The parsed `MmprojConfig.projector` is the upstream-most
        // signal (`clip.cpp:865-867` gates the qwen3vl builder on it).
        let arch = crate::inference::vision::mmproj::detect_arch_profile_with_projector(
            &mmp_config.projector,
            &actual_names,
        );
        if !arch.is_supported() {
            anyhow::bail!(
                "mmproj arch profile is Unknown — neither Gemma 4 \
                 SigLIP markers (ln1/ln2/post_ffw_norm) nor CLIP marker \
                 (attn_norm) found in block 0. hf2q's ViT forward pass \
                 cannot dispatch on this file."
            );
        }
        // Load every tensor onto the Metal device. For Gemma 4 this is
        // ~400MB / 356 tensors / ~10s cold-cache on M5 Max.
        //
        // Iter-103 added the `HF2Q_SKIP_MMPROJ_LOAD=1` escape hatch
        // for bisecting the chat-warmup-logits-go-NaN bug: if
        // skipping the mmproj weight load (just keep the config +
        // arch detection) makes chat warmup produce valid logits,
        // the bug is in `LoadedMmprojWeights::load`'s buffer-alloc
        // / dequant path; if NaN persists, the bug is somewhere
        // earlier (the GGUF mmap itself).
        let skip_mmproj_load = std::env::var("HF2Q_SKIP_MMPROJ_LOAD").as_deref() == Ok("1");
        let device = mlx_native::MlxDevice::new()
            .map_err(|e| anyhow::anyhow!("create MlxDevice for mmproj load: {e}"))?;
        let mmp_weights = if skip_mmproj_load {
            tracing::warn!(
                "HF2Q_SKIP_MMPROJ_LOAD=1 — using empty mmproj weights; \
                 vision requests will 500 on first forward attempt"
            );
            crate::inference::vision::mmproj_weights::LoadedMmprojWeights::empty(device)
        } else {
            crate::inference::vision::mmproj_weights::LoadedMmprojWeights::load(
                &gguf,
                &mmp_config,
                device,
            )
            .map_err(|e| anyhow::anyhow!("mmproj weight load: {e}"))?
        };
        let model_id = mmp_path
            .file_stem()
            .map(|s| s.to_string_lossy().into_owned())
            .unwrap_or_else(|| "mmproj".into());
        tracing::info!(
            path = %mmp_path.display(),
            image_size = mmp_config.image_size,
            patch_size = mmp_config.patch_size,
            hidden = mmp_config.hidden_size,
            layers = mmp_config.num_hidden_layers,
            projector = mmp_config.projector.as_str(),
            arch = arch.as_str(),
            tensors_loaded = mmp_weights.len(),
            "Loaded mmproj GGUF header + tensor set + weights"
        );
        // Iter 53: ViT GPU warmup — runs one synthetic full forward
        // to trigger Metal kernel pipeline compilation. Drops first
        // user-visible multimodal request from ~5–10s (cold compile)
        // to ~1.3s (steady-state) on M5 Max.
        //
        // Iter-103 added the `HF2Q_SKIP_VIT_WARMUP=1` escape hatch
        // for bisecting the chat-warmup-logits-go-NaN bug: if
        // skipping the ViT warmup makes chat warmup produce valid
        // logits, the bug lives in `warmup_vit_gpu`'s leftover GPU
        // state; if NaN persists, the bug lives in
        // `LoadedMmprojWeights::load`.
        let skip_vit_warmup = std::env::var("HF2Q_SKIP_VIT_WARMUP").as_deref() == Ok("1");
        if skip_vit_warmup {
            tracing::warn!(
                "HF2Q_SKIP_VIT_WARMUP=1 — skipping ViT GPU warmup; first \
                 multimodal request will pay kernel-compile cost"
            );
        } else {
            let warmup_t0 = std::time::Instant::now();
            match crate::inference::vision::vit_gpu::warmup_vit_gpu(&mmp_weights, &mmp_config) {
                Ok(()) => tracing::info!(
                    elapsed_ms = warmup_t0.elapsed().as_millis() as u64,
                    "ViT GPU warmup complete"
                ),
                Err(e) => tracing::warn!(
                    error = %e,
                    "ViT GPU warmup failed; first multimodal request will pay kernel-compile cost"
                ),
            }
        }
        Some(api::state::LoadedMmproj {
            gguf_path: mmp_path.clone(),
            config: mmp_config,
            arch,
            weights: std::sync::Arc::new(mmp_weights),
            model_id,
        })
    } else {
        None
    };

    // --- Build router ---
    // `state` was constructed above (iter-209) with the cache + hardware
    // + empty pool; pre-warm has already admitted the `--model` engine
    // (when supplied).  Here we attach the embedding model + mmproj
    // descriptors before the router takes ownership.
    if let Some(em) = embedding_model {
        // Pre-warm a persistent kernel registry: register all kernels
        // the arch needs + run one warmup forward against the loaded
        // weights so every Metal pipeline compiles and caches before
        // the first /v1/embeddings request. Eliminates the ~150 ms
        // per-request shader-compile cost surfaced by iter-82
        // benchmarking. Stashes the registry behind an Arc<Mutex<>>
        // for handler dispatch.
        let registry = build_warmed_embedding_registry(&em).context("warm embedding registry")?;
        state = state
            .with_embedding_model(em)
            .with_embedding_registry(std::sync::Arc::new(std::sync::Mutex::new(registry)));
    }
    if let Some(m) = mmproj {
        state = state.with_mmproj(m);
    }
    let state_for_warmup = state.clone();
    let router = api::build_router(state);

    // --- Async runtime + serve ---
    let rt = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .build()
        .context("building tokio runtime")?;

    let stdout_is_tty = std::io::IsTerminal::is_terminal(&std::io::stdout());
    let quiet = args.quiet;

    rt.block_on(async move {
        let bind = format!("{}:{}", config.host, config.port);
        let listener = tokio::net::TcpListener::bind(&bind)
            .await
            .with_context(|| format!("binding to {bind}"))?;
        let local_addr = listener.local_addr().ok();
        tracing::info!(
            addr = %local_addr.map(|a| a.to_string()).unwrap_or_else(|| bind.clone()),
            "hf2q HTTP server listening"
        );
        if let Some(engine) = startup_engine_for_banner.as_ref() {
            let mut stdout = std::io::stdout();
            maybe_print_serve_banner(engine.info(), &mut stdout, stdout_is_tty, quiet)
                .context("print serve load banner")?;
        }
        // ADR-017 Closure iter-10 (2026-05-05): drop the banner-only
        // Engine clone NOW. Without this, the clone (an `Arc<EngineInner>`
        // ref) lived for the entire `rt.block_on` async block — i.e. the
        // whole server lifetime — which kept the canonical model's
        // `EngineInner` alive even after the pool's LRU eviction dropped
        // the manager's `Arc<LoadedEngine>`. The worker thread + its
        // `LoadedModel` (multi-GB MlxModelWeights) never released, leaking
        // ~15 GB of anon memory per stress iter (verified iter-10c stress
        // smoke: cache_dir=13.9 GB but RSS=42 GB; the canonical engine
        // wasn't being freed by eviction). Banner only needed `info()`
        // which is a `&LoadInfo` borrow — we don't need the engine
        // afterwards.
        drop(startup_engine_for_banner);
        eprintln!("hf2q serving on http://{}", bind);

        // Iter-209: warmup ran SYNCHRONOUSLY at pre-warm time (when
        // `--model` was supplied) inside `pool.load_or_get` →
        // `DefaultModelLoader::load` → `load_engine` (warmup_synchronously
        // = true).  ready_for_gen is already initialized to `true` by
        // `AppState::new_for_serve`; this block remains as a no-op
        // observability anchor — the previous engine.is_some() guard is
        // replaced with a pool-state log so operators see the boot
        // ordering signal in the logs.  The iter-103 ordering invariant
        // (chat warmup BEFORE mmproj load) is preserved by the call
        // ordering above (pre-warm runs before mmproj load).
        {
            let pool_state_log = state_for_warmup.pool.read().ok().map(|m| m.pool_stats());
            if let Some(stats) = pool_state_log {
                tracing::info!(
                    loaded = stats.loaded_count,
                    capacity = stats.capacity_models,
                    bytes_resident = stats.total_resident_bytes,
                    bytes_budget = stats.memory_budget_bytes,
                    "hf2q ready (pool-backed; pre-warm complete if --model supplied)"
                );
            }
        }

        axum::serve(listener, router)
            .with_graceful_shutdown(shutdown_signal())
            .await
            .context("axum::serve")?;

        // Axum has stopped accepting + drained in-flight HTTP responses.
        // Each in-flight handler that called `engine.generate*` has
        // already received its reply.  Iter-209: with the pool replacing
        // the single-slot Option<Engine>, we shut down EVERY pooled
        // engine in parallel (each owns a separate worker thread + its
        // own GPU resources).  Without this, the tokio runtime drops at
        // the bottom of `block_on` and the `mpsc::Sender` to each worker
        // is closed implicitly; the worker exits its loop on the next
        // `blocking_recv`, but a mid-decode generation gets cut off
        // rather than running to its natural finish_reason.
        //
        // ORDERING (ADR-017 Closure iter-2): snapshot engine Arcs
        // FIRST, then run the KV-cache drain (which evicts entries from
        // the pool's manager, dropping its Arcs — the snapshot keeps
        // the engines alive through the drain). After drain, run
        // explicit `engine.shutdown().await` per snapshot Arc to clean
        // up the worker threads.
        let shutdown_engines: Vec<_> = state_for_warmup
            .pool
            .read()
            .ok()
            .map(|mgr| {
                mgr.snapshot_engines()
                    .into_iter()
                    .map(|le| le.engine.clone())
                    .collect()
            })
            .unwrap_or_default();

        // ADR-017 Closure iter-2 (2026-05-04): graceful-shutdown spill.
        // SIGTERM / SIGINT (or `POST /shutdown`, which calls
        // `libc::raise(SIGTERM)` on itself — see
        // `api::handlers::shutdown`) fired the signal that
        // `shutdown_signal()` was awaiting.  Now walk the loaded pool,
        // fire `pre_evict` per entry to push block-aligned KV state
        // into the async writer queue, and poll until the writer
        // drains (or 30 s elapses).  Without this the in-memory KV
        // cache is silently lost across server restarts — breaks
        // Ctrl+C / `systemctl stop` / deploy semantics, and makes
        // ADR-017 R-P5 ("ssd_cold_post_restart" cache state)
        // unreachable without a prior model-swap eviction.  Off-path
        // (no `--kv-persist`) `state.kv_spiller` is `None` and the
        // helper returns immediately with an all-zero summary.
        //
        // ADR-017 Closure iter-3 (2026-05-04): wrap in
        // `tokio::task::spawn_blocking` because the drain helper is
        // synchronous and ends up calling `Engine::request_kv_snapshot`
        // (engine.rs:2335) which uses `tokio::block_on` to wait for the
        // worker reply. Calling `block_on` from inside the runtime's
        // own async context panics with "Cannot block the current
        // thread from within a runtime." The blocking-thread-pool
        // path is the canonical Tokio pattern for running sync
        // blocking code from async context — block_in_place would
        // also work but spawn_blocking keeps the runtime threads
        // available for axum's outstanding shutdown work.
        let state_for_drain = state_for_warmup.clone();
        let drain_summary = tokio::task::spawn_blocking(move || {
            drain_loaded_models_to_disk(&state_for_drain, std::time::Duration::from_secs(30))
        })
        .await
        .unwrap_or_default();
        tracing::info!(
            evicted = drain_summary.evicted,
            drain_ms = drain_summary.drain_ms,
            queue_depth_at_exit = drain_summary.queue_depth_at_exit,
            timed_out = drain_summary.timed_out,
            "ADR-017 graceful-shutdown KV-cache drain complete"
        );

        for engine in shutdown_engines {
            match engine.shutdown().await {
                Ok(()) => tracing::info!("hf2q-engine worker joined"),
                Err(e) => tracing::warn!(error = %e, "hf2q-engine worker join failed"),
            }
        }

        tracing::info!("hf2q HTTP server shut down cleanly");
        Ok::<(), anyhow::Error>(())
    })?;
    Ok(())
}

/// Build the server's `system_fingerprint` — `hf2q-<short-git-sha-or-ver>-mlx-native`.
fn system_fingerprint() -> String {
    // Prefer the CARGO_PKG_VERSION (baked at build time). Git sha could be
    // added via a build.rs; for now the pkg version + backend is sufficient
    // identity for OpenAI's `system_fingerprint` contract.
    format!("hf2q-{}-mlx-native", env!("CARGO_PKG_VERSION"))
}

/// ADR-017 Closure iter-2 (2026-05-04): summary of the
/// graceful-shutdown KV-cache drain returned by
/// [`drain_loaded_models_to_disk`]. Logged at INFO and surfaced to
/// HTTP callers via the `POST /shutdown` handler so monitoring / CI
/// pipelines can confirm the cache state at exit.
#[derive(Debug, Clone, Default)]
pub struct ShutdownDrainSummary {
    /// Number of `(repo, quant)` pool entries that received a
    /// `pre_evict` call. Each evicted entry's spill outcome is
    /// already counted in `hf2q_pool_kv_spills_total{outcome=...}`
    /// by the existing trigger sites; this field is the raw count
    /// of `pre_evict` invocations regardless of per-entry outcome.
    pub evicted: usize,
    /// Wall-time milliseconds from the first `evict` call to the
    /// final `pending_writer_queue_depth() == 0` observation (or
    /// to the timeout deadline, whichever came first).
    pub drain_ms: u64,
    /// Writer queue depth observed at function exit. `0` indicates
    /// a clean drain; non-zero indicates the timeout fired before
    /// the async writer caught up (operator should bump
    /// `--shutdown-drain-secs` or investigate writer-thread
    /// throughput).
    pub queue_depth_at_exit: usize,
    /// `true` if the drain hit the deadline before the queue
    /// emptied. Off-path (no kv-persist) returns `false` (queue
    /// trivially empty, no timeout possible).
    pub timed_out: bool,
}

/// ADR-017 Closure iter-2 (2026-05-04): graceful-shutdown KV-cache
/// drain. Walks the loaded pool, fires `pre_evict` on every entry
/// to flush block-aligned K/V state into the async writer queue,
/// then polls [`BlockPrefixCacheSpiller::pending_writer_queue_depth`]
/// until it reaches zero or `timeout` elapses.
///
/// **Why this exists (Chesterton's fence read of ADR-017's R-F2):**
/// the spiller's only production trigger sites are eviction-driven
/// (`MultiModelManager::evict` at `multi_model.rs:1090`, LRU-driven
/// admission evict at `:1189`). There is no in-flight write-through,
/// no periodic background flush, no graceful-shutdown spill in the
/// pre-iter-2 architecture. So a `kill <pid>` / Ctrl-C / deploy
/// silently discards the entire in-memory KV cache, and ADR-017
/// R-P5 ("ssd_cold_post_restart" cache state) is unreachable
/// without a prior model-swap eviction. This drain closes that gap
/// at the graceful-shutdown boundary.
///
/// Called from two paths that converge on the same drain semantics
/// (Option β chosen 2026-05-04):
/// 1. `cmd_serve`'s post-`axum::serve` block — runs after SIGTERM /
///    Ctrl-C / SIGINT delivery + axum's graceful-shutdown path.
///    Catches `kill <pid>`, `Ctrl-C`, `systemctl stop`,
///    `launchctl stop`.
/// 2. `POST /shutdown` HTTP handler — calls `libc::raise(SIGTERM)`
///    on itself so the same signal-driven path runs. This gives
///    orchestrators that prefer HTTP (k8s pre-stop hooks, monitoring
///    tooling, R-P5 test harness) the same drain semantics as Unix
///    signals.
///
/// **Off-path (no `--kv-persist`):** `state.kv_spiller` is `None`
/// and this function returns immediately with an all-zero
/// `ShutdownDrainSummary`. The pool is NOT evicted in that case —
/// the existing post-axum engine.shutdown() loop is the only
/// shutdown path needed when kv-persist is disabled.
///
/// **Lock semantics:** the read lock is held only long enough to
/// snapshot `(repo, quant)` keys, then dropped before the write
/// lock is acquired. Holding read+write across the boundary would
/// deadlock if any in-flight handler is still waking up after
/// axum's graceful-shutdown.
pub fn drain_loaded_models_to_disk(
    state: &api::state::AppState,
    timeout: std::time::Duration,
) -> ShutdownDrainSummary {
    let Some(spiller) = state.kv_spiller.as_ref() else {
        // kv-persist disabled at startup; nothing to drain.
        return ShutdownDrainSummary::default();
    };

    let start = std::time::Instant::now();

    // 1. Snapshot the loaded (repo, quant) keys under the read lock,
    //    then drop it before acquiring the write lock for evictions.
    let keys: Vec<(String, crate::serve::quant_select::QuantType)> = state
        .pool
        .read()
        .ok()
        .map(|mgr| {
            mgr.snapshot_engines()
                .into_iter()
                .map(|le| (le.repo.clone(), le.quant))
                .collect()
        })
        .unwrap_or_default();

    let evicted = if keys.is_empty() {
        0
    } else if let Ok(mut mgr) = state.pool.write() {
        let mut count = 0usize;
        for (repo, quant) in &keys {
            // evict() fires pre_evict on the spiller (which enqueues
            // block writes) BEFORE removing the entry from the pool's
            // manager.engines map. The spiller's outcome is already
            // recorded in hf2q_pool_kv_spills_total{outcome=...} by the
            // existing trigger site (multi_model.rs:1090).
            let _ = mgr.evict(repo, *quant);
            count += 1;
        }
        count
    } else {
        // Lock poisoned; bail with zero progress and let the caller
        // observe queue_depth_at_exit > 0 if any blocks were already
        // in flight.
        0
    };

    // 2. Poll the async writer queue until drained or timeout.
    let poll_interval = std::time::Duration::from_millis(50);
    let mut queue_depth_at_exit = spiller.pending_writer_queue_depth();
    let mut timed_out = false;
    while queue_depth_at_exit > 0 {
        if start.elapsed() >= timeout {
            timed_out = true;
            break;
        }
        std::thread::sleep(poll_interval);
        queue_depth_at_exit = spiller.pending_writer_queue_depth();
    }

    let drain_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
    ShutdownDrainSummary {
        evicted,
        drain_ms,
        queue_depth_at_exit,
        timed_out,
    }
}

/// Graceful-shutdown signal handler: wait for Ctrl-C or SIGTERM (Decision #17).
async fn shutdown_signal() {
    let ctrl_c = async {
        let _ = tokio::signal::ctrl_c().await;
    };
    #[cfg(unix)]
    let terminate = async {
        if let Ok(mut s) = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
        {
            s.recv().await;
        }
    };
    #[cfg(not(unix))]
    let terminate = std::future::pending::<()>();
    tokio::select! {
        _ = ctrl_c => tracing::info!("received SIGINT, shutting down"),
        _ = terminate => tracing::info!("received SIGTERM, shutting down"),
    }
}

/// Run the `cache` subcommand (ADR-005 Phase 3 iter-205, AC line 5351).
///
/// Three actions:
/// - `cache list`  — enumerate cached models + quants + on-disk size.
/// - `cache size`  — total bytes used by the cache.
/// - `cache clear` — invalidate entries (single quant, whole repo, or all).
///
/// ADR-017 §R-F6: each action accepts `--kv-namespace` to retarget
/// from the model-weights cache to the persistent KV cache subtree
/// (`<kv-persist>/models/<fp_short>/`). Path discovery uses
/// `--kv-path PATH` then `HF2Q_KV_PERSIST_PATH`; no implicit default.
/// See `docs/operating-kv-cache.md` §11 #4.
///
/// Output is human-readable text on stdout; errors go through stderr
/// via the standard `Result` plumbing.  Bytes-freed is reported back
/// to the operator from the on-disk walk performed before removal.
///
/// Safety:
/// - `--all` requires `--yes` (refuses with a named error otherwise).
/// - Without `--model` and without `--all`, prints the usage and
///   exits with `Err` (not a silent no-op — the operator likely
///   intended one of the two).
/// - `--kv-namespace clear` refuses without `--model` and refuses
///   `--all` entirely (per-repo scope only — see runbook §11 #4).
/// - `--kv-namespace clear` consults a `<kv_root>/.cache_lock`
///   sentinel via `flock(LOCK_EX | LOCK_NB)`; pass `--force` to
///   override.
pub fn cmd_cache(args: cli::CacheArgs) -> Result<()> {
    use cli::CacheAction;

    // ADR-017 §R-F6: when `--kv-namespace` is set on any action, the
    // operation targets `<kv-persist>/models/` instead of the
    // weights-side `cache::ModelCache`. We dispatch BEFORE opening
    // the weights cache so a kv-namespace op never touches
    // `~/.cache/hf2q/cache_index.json`.
    match args.action {
        CacheAction::List {
            kv_namespace: true,
            kv_path,
        } => {
            return cmd_cache_kv_list(kv_path.as_deref());
        }
        CacheAction::Size {
            kv_namespace: true,
            kv_path,
        } => {
            return cmd_cache_kv_size(kv_path.as_deref());
        }
        CacheAction::Clear {
            kv_namespace: true,
            ref model,
            ref quant,
            all,
            yes: _,
            ref kv_path,
            force,
        } => {
            return cmd_cache_kv_clear(
                kv_path.as_deref(),
                model.as_deref(),
                quant.as_deref(),
                all,
                force,
            );
        }
        _ => { /* fall through to weights-side */ }
    }

    let mut cache = cache::ModelCache::open().context("open model cache")?;
    match args.action {
        CacheAction::List { .. } => cmd_cache_list(&cache),
        CacheAction::Size { .. } => cmd_cache_size(&cache),
        CacheAction::Clear {
            model,
            quant,
            all,
            yes,
            kv_namespace: _,
            kv_path: _,
            force: _,
        } => cmd_cache_clear(&mut cache, model, quant, all, yes),
    }
}

/// `hf2q cache list --kv-namespace [--kv-path PATH]` — ADR-017 §R-F6.
///
/// Walks `<kv_root>/models/` and emits one row per `<fp_short>` dir
/// with `bytes_on_disk` + `block_count`. Tolerates missing
/// `<kv_root>` / missing `models/` (operator may invoke before the
/// first `cmd_serve --kv-persist` run).
fn cmd_cache_kv_list(kv_path: Option<&Path>) -> Result<()> {
    use crate::serve::kv_persist::cache_ops;

    let kv_root = cache_ops::resolve_kv_root(kv_path)?;
    let entries = cache_ops::list_namespaces(&kv_root)?;
    if entries.is_empty() {
        println!("(kv-cache empty — root: {})", kv_root.display());
        return Ok(());
    }
    println!("hf2q kv-cache @ {}", kv_root.display());
    println!(
        "{:<20} {:>14} {:>10}",
        "FP_SHORT", "BYTES_ON_DISK", "BLOCKS"
    );
    for e in &entries {
        println!(
            "{:<20} {:>14} {:>10}",
            e.fp_short, e.bytes_on_disk, e.block_count
        );
    }
    Ok(())
}

/// `hf2q cache size --kv-namespace [--kv-path PATH]` — ADR-017 §R-F6.
fn cmd_cache_kv_size(kv_path: Option<&Path>) -> Result<()> {
    use crate::serve::kv_persist::cache_ops;

    let kv_root = cache_ops::resolve_kv_root(kv_path)?;
    let total = cache_ops::total_bytes(&kv_root);
    println!(
        "hf2q kv-cache @ {} — {} bytes ({:.2} GiB)",
        kv_root.display(),
        total,
        total as f64 / (1u64 << 30) as f64,
    );
    Ok(())
}

/// `hf2q cache clear --kv-namespace --model <repo> [--quant <q>]
///   [--kv-path PATH] [--force]` — ADR-017 §R-F6.
///
/// Refuses without `--model` (no whole-cache wipe — operator runbook
/// §11 #4 deliberately scopes this command to per-repo); refuses
/// under `--all` (likewise). Without `--force` the active-serve
/// sentinel-flock guard is consulted; the diagnostic is explicit
/// about its meaning.
fn cmd_cache_kv_clear(
    kv_path: Option<&Path>,
    model: Option<&str>,
    quant: Option<&str>,
    all: bool,
    force: bool,
) -> Result<()> {
    use crate::serve::kv_persist::cache_ops;
    use crate::serve::quant_select::QuantType;

    if all {
        return Err(anyhow::anyhow!(
            "hf2q cache clear --kv-namespace: --all is not supported. \
             Per-repo scope only (operator runbook §11 #4). To wipe \
             the entire kv-cache, stop `hf2q serve` and `rm -rf \
             <kv-persist>/models <kv-persist>/locks` directly."
        ));
    }
    let Some(repo) = model else {
        return Err(anyhow::anyhow!(
            "hf2q cache clear --kv-namespace: --model <repo-id> required \
             (no whole-cache wipe via this command — see operator runbook §11 #4)"
        ));
    };

    let kv_root = cache_ops::resolve_kv_root(kv_path)?;

    if let Some(q_str) = quant {
        let q =
            QuantType::from_canonical_str(q_str).map_err(|e| anyhow::anyhow!("--quant: {}", e))?;
        let outcome = cache_ops::clear_namespace(&kv_root, repo, q, force).map_err(|e| {
            anyhow::anyhow!(
                "hf2q cache clear --kv-namespace --model {} --quant {}: {}",
                repo,
                q.as_str(),
                e
            )
        })?;
        if outcome.existed {
            println!(
                "hf2q kv-cache: cleared {}@{} (fp_short={}, {} bytes freed)",
                repo,
                q.as_str(),
                outcome.fp_short,
                outcome.bytes_freed
            );
        } else {
            println!(
                "hf2q kv-cache: nothing to clear for {}@{} (fp_short={} not present)",
                repo,
                q.as_str(),
                outcome.fp_short
            );
        }
    } else {
        let outcomes =
            cache_ops::clear_namespace_all_quants(&kv_root, repo, force).map_err(|e| {
                anyhow::anyhow!(
                    "hf2q cache clear --kv-namespace --model {} (all quants): {}",
                    repo,
                    e
                )
            })?;
        let total_bytes: u64 = outcomes.iter().map(|o| o.bytes_freed).sum();
        let removed: Vec<&str> = outcomes
            .iter()
            .filter(|o| o.existed)
            .map(|o| o.fp_short.as_str())
            .collect();
        if removed.is_empty() {
            println!(
                "hf2q kv-cache: nothing to clear for {} (no quant variants present)",
                repo
            );
        } else {
            println!(
                "hf2q kv-cache: cleared {} (all quants) — {} fp_short dirs, {} bytes freed: [{}]",
                repo,
                removed.len(),
                total_bytes,
                removed.join(", ")
            );
        }
    }
    Ok(())
}

fn cmd_cache_list(cache: &cache::ModelCache) -> Result<()> {
    let entries: Vec<_> = cache.iter_entries().collect();
    if entries.is_empty() {
        println!("(cache empty — root: {})", cache.root().display());
        return Ok(());
    }
    println!("hf2q cache @ {}", cache.root().display());
    println!(
        "{:<48} {:<10} {:>12} {:>20}",
        "MODEL", "QUANT", "BYTES", "LAST_ACCESSED"
    );
    for view in &entries {
        if view.model.quantizations.is_empty() {
            // Source recorded but no quants yet (an in-flight or
            // failed quantize); render the model row with `(none)`
            // so it's visible to `cache list`.
            println!(
                "{:<48} {:<10} {:>12} {:>20}",
                view.repo_id, "(none)", "-", view.model.last_accessed_secs,
            );
            continue;
        }
        for (quant, qe) in &view.model.quantizations {
            println!(
                "{:<48} {:<10} {:>12} {:>20}",
                view.repo_id, quant, qe.bytes, view.model.last_accessed_secs,
            );
        }
    }
    Ok(())
}

fn cmd_cache_size(cache: &cache::ModelCache) -> Result<()> {
    let total = cache.total_bytes_on_disk();
    println!(
        "hf2q cache @ {} — {} bytes ({:.2} GiB)",
        cache.root().display(),
        total,
        total as f64 / (1u64 << 30) as f64,
    );
    Ok(())
}

fn cmd_cache_clear(
    cache: &mut cache::ModelCache,
    model: Option<String>,
    quant: Option<String>,
    all: bool,
    yes: bool,
) -> Result<()> {
    use crate::serve::quant_select::QuantType;

    // Sanity: cannot mix --all with --model / --quant.  Refusing here
    // (rather than silently picking one path) prevents an operator
    // from running `cache clear --model x --all --yes` and being
    // surprised which one won.
    if all && (model.is_some() || quant.is_some()) {
        return Err(anyhow::anyhow!(
            "hf2q cache clear: --all is mutually exclusive with --model / --quant"
        ));
    }
    if !all && model.is_none() {
        return Err(anyhow::anyhow!(
            "hf2q cache clear: must specify --model <repo-id> [--quant <type>] \
             OR --all --yes (the latter purges every cached model)"
        ));
    }

    if all {
        if !yes {
            return Err(anyhow::anyhow!(
                "hf2q cache clear --all: refused without --yes \
                 (this would remove every cached model under {})",
                cache.root().display()
            ));
        }
        let freed = cache.purge().context("purge cache")?;
        println!(
            "hf2q cache: purged ({} bytes / {:.2} GiB freed)",
            freed,
            freed as f64 / (1u64 << 30) as f64,
        );
        return Ok(());
    }

    // --model is set (validated above).  --quant optional.
    let repo = model.expect("validated above");
    if let Some(q_str) = quant {
        let q =
            QuantType::from_canonical_str(&q_str).map_err(|e| anyhow::anyhow!("--quant: {}", e))?;
        let freed = cache
            .invalidate(&repo, q)
            .with_context(|| format!("clear {}@{}", repo, q.as_str()))?;
        println!(
            "hf2q cache: cleared {}@{} ({} bytes freed)",
            repo,
            q.as_str(),
            freed
        );
    } else {
        let freed = cache
            .invalidate_repo(&repo)
            .with_context(|| format!("clear {} (all quants)", repo))?;
        println!(
            "hf2q cache: cleared {} (all quants — {} bytes freed)",
            repo, freed
        );
    }
    Ok(())
}

/// Run the `parity` subcommand (ADR-009 Phase 2).
///
/// `parity check` — compare hf2q output against locked reference fixtures.
/// `parity capture` — generate fresh reference outputs from hf2q.
pub fn cmd_parity(args: cli::ParityArgs) -> Result<()> {
    use cli::ParityCommand;

    match args.command {
        ParityCommand::Check {
            model,
            prompt,
            min_prefix,
            max_tokens,
            self_baseline,
            tq_quality,
            fixture,
            cosine_mean_floor,
            cosine_p1_floor,
            argmax_max,
            ppl_delta_max,
        } => {
            if tq_quality {
                // ADR-007 §853-866 Gate H — TQ-active envelope check.
                // The fixture is required: Gate H is a comparison gate
                // (cosine / argmax / PPL Δ vs the frozen
                // <prompt>_tq_quality.json), not absolute.  Erroring
                // early without --fixture beats running a 1000-token
                // two-regime decode and then discovering nothing to
                // compare against.
                if self_baseline {
                    anyhow::bail!(
                        "parity check --tq-quality is incompatible with \
                         --self-baseline (Gate D vs Gate H — different gates)"
                    );
                }
                let fixture = fixture.ok_or_else(|| {
                    anyhow::anyhow!(
                        "parity check --tq-quality requires --fixture \
                         <path/to/<prompt>_tq_quality.json>.\n\
                         Hint: the fixture is produced by `hf2q parity \
                         capture --tq-quality --model <gguf> --prompt \
                         {prompt}` (iter-112)."
                    )
                })?;
                parity_quality::cmd_parity_check_tq_quality(
                    &model,
                    &prompt,
                    &fixture,
                    cosine_mean_floor,
                    cosine_p1_floor,
                    argmax_max,
                    ppl_delta_max,
                    max_tokens,
                )
            } else {
                // Suppress unused-warnings for the Gate H args on the
                // byte-prefix path.
                let _ = (
                    fixture,
                    cosine_mean_floor,
                    cosine_p1_floor,
                    argmax_max,
                    ppl_delta_max,
                );
                cmd_parity_check(&model, &prompt, min_prefix, max_tokens, self_baseline)
            }
        }
        ParityCommand::Capture {
            model,
            output,
            prompt,
            max_tokens,
            tq_quality,
        } => {
            if tq_quality {
                parity_quality::cmd_parity_capture_tq_quality(&model, &output, &prompt, max_tokens)
            } else {
                cmd_parity_capture(&model, &output, &prompt, max_tokens)
            }
        }
    }
}

/// Parity check: run hf2q on a prompt and compare against locked reference.
fn cmd_parity_check(
    model_path: &Path,
    prompt_name: &str,
    min_prefix: Option<usize>,
    max_tokens: Option<usize>,
    self_baseline: bool,
) -> Result<()> {
    let evals_dir = Path::new("tests/evals");
    let ref_dir = evals_dir.join("reference");

    // Load prompt
    let prompt_file = evals_dir.join("prompts").join(format!("{prompt_name}.txt"));
    anyhow::ensure!(
        prompt_file.exists(),
        "Prompt file not found: {}",
        prompt_file.display()
    );
    let prompt_text = std::fs::read_to_string(&prompt_file)?.trim().to_string();

    // Load reference. Default: llama.cpp-anchored parity (*_llama.txt).
    // Gate D (--self-baseline): hf2q frozen self-baseline (*_hf2q.txt),
    // bisect-safe when math deliberately changes and llama.cpp drift is
    // expected.
    let ref_suffix = if self_baseline { "_hf2q" } else { "_llama" };
    let ref_file = ref_dir.join(format!("{prompt_name}{ref_suffix}.txt"));
    anyhow::ensure!(
        ref_file.exists(),
        "Reference file not found: {}",
        ref_file.display()
    );
    let ref_bytes = std::fs::read(&ref_file)?;

    // Determine settings
    let manifest: serde_json::Value =
        serde_json::from_str(&std::fs::read_to_string(ref_dir.join("MANIFEST.json"))?)?;
    let prompt_meta = &manifest["prompts"][prompt_name];
    let tokens =
        max_tokens.unwrap_or_else(|| prompt_meta["max_tokens"].as_u64().unwrap_or(1000) as usize);
    let threshold = min_prefix.unwrap_or_else(||
        // Parse from gate field like "common_prefix >= 3094"
        prompt_meta["parity_gate"].as_str()
            .and_then(|s| s.split(">=").nth(1))
            .and_then(|s| s.trim().parse::<usize>().ok())
            .unwrap_or(0));

    // Run hf2q
    eprintln!("=== Parity Check: {} ===", prompt_name);
    eprintln!("Model:     {}", model_path.display());
    eprintln!("Prompt:    {} ({} chars)", prompt_name, prompt_text.len());
    eprintln!("Tokens:    {}", tokens);
    eprintln!("Threshold: {} bytes", threshold);
    eprintln!();

    let tokenizer_path = find_tokenizer(model_path, None)?;
    let mut ctx = gpu::GpuContext::new().map_err(|e| anyhow::anyhow!("GPU init: {e}"))?;
    let gguf = mlx_native::gguf::GgufFile::open(model_path)
        .map_err(|e| anyhow::anyhow!("GGUF open: {e}"))?;
    // ADR-022 P1.8 (operator pushback "requiring the config seems dumb"):
    // the GGUF carries every architectural parameter `Gemma4Config` needs;
    // there is no reason for parity check to require a sibling HF config.json
    // when the GGUF is the source of truth.  `from_gguf` matches the
    // `from_config_json` defaults so configs are byte-identical.
    let cfg = config::Gemma4Config::from_gguf(&gguf)?;
    // cmd_parity has its own output contract — no progress line.
    let mut parity_progress = header::LoadProgress::new(false, 1, 0);
    let mut mlx_w = crate::inference::models::gemma4::MlxModelWeights::load_from_gguf(
        &gguf,
        &cfg,
        &mut ctx,
        &mut parity_progress,
    )?;

    let mut tokenizer = tokenizers::Tokenizer::from_file(&tokenizer_path)
        .map_err(|e| anyhow::anyhow!("Tokenizer: {e}"))?;
    tokenizer
        .with_truncation(None)
        .map_err(|e| anyhow::anyhow!("Tokenizer truncation: {e}"))?;

    let rendered = render_chat_template(
        &gguf,
        &cli::GenerateArgs {
            model: model_path.to_path_buf(),
            prompt: Some(prompt_text.clone()),
            prompt_file: None,
            tokenizer: None,
            config: None,
            temperature: 0.0,
            top_p: 1.0,
            top_k: 0,
            min_p: 0.0,
            repetition_penalty: 1.0,
            max_tokens: tokens,
            mmproj: None,
            image: None,
            chat_template: None,
            chat_template_file: None,
            benchmark: false,
            speculative: false,
            kv_bits: None,
            enable_thinking: false,
            no_thinking: false,
            ignore_eos: false,
        },
        Some(&tokenizer),
        &prompt_text,
    )?;

    let encoding = tokenizer
        .encode(rendered.as_str(), false)
        .map_err(|e| anyhow::anyhow!("Tokenize: {e}"))?;
    let prompt_tokens: Vec<u32> = encoding.get_ids().to_vec();

    // Prefill + decode
    let eos_token_ids: Vec<u32> = vec![1, 106];
    let first_token = mlx_w.forward_prefill(&prompt_tokens, tokens, &mut ctx)?;
    let mut all_tokens = prompt_tokens.to_vec();
    let mut next_token = first_token;
    all_tokens.push(next_token);

    for _ in 1..tokens {
        if eos_token_ids.contains(&next_token) {
            break;
        }
        let pos = all_tokens.len() - 1;
        let mut p = None;
        next_token = mlx_w.forward_decode(next_token, pos, &mut ctx, &mut p)?;
        all_tokens.push(next_token);
    }

    // Decode generated tokens to text
    let gen_tokens = &all_tokens[prompt_tokens.len()..];
    let hf2q_text = tokenizer.decode(gen_tokens, false).unwrap_or_default();
    let hf2q_bytes = hf2q_text.as_bytes();

    // Compare
    let n = ref_bytes.len().min(hf2q_bytes.len());
    let mut common = 0;
    while common < n && ref_bytes[common] == hf2q_bytes[common] {
        common += 1;
    }

    eprintln!();
    let ref_label = if self_baseline {
        "frozen hf2q"
    } else {
        "llama.cpp"
    };
    println!("Reference: {} bytes ({})", ref_bytes.len(), ref_label);
    println!("hf2q:      {} bytes", hf2q_bytes.len());
    println!("Common:    {} bytes", common);
    if self_baseline {
        // Gate D contract: byte-identical. Length must match AND every
        // byte in the common-prefix comparison was equal.
        let identical = hf2q_bytes.len() == ref_bytes.len() && common == ref_bytes.len();
        if identical {
            println!(
                "PASS: byte-identical to frozen hf2q baseline ({} bytes)",
                common
            );
        } else {
            println!("FAIL: not byte-identical to frozen hf2q baseline");
            if common < n {
                let ctx_start = common;
                let ctx_end = (common + 80).min(n);
                let ref_snip = String::from_utf8_lossy(&ref_bytes[ctx_start..ctx_end]);
                let hf2q_snip =
                    String::from_utf8_lossy(&hf2q_bytes[ctx_start..ctx_end.min(hf2q_bytes.len())]);
                println!();
                println!("Divergence at byte {}:", common);
                println!("  frozen: {:?}", ref_snip);
                println!("  hf2q:   {:?}", hf2q_snip);
            }
            anyhow::bail!("Self-baseline check failed: hf2q differs from frozen baseline");
        }
    } else {
        println!("Threshold: {} bytes", threshold);
        if common >= threshold {
            println!("PASS: {} >= {}", common, threshold);
            if common > threshold {
                println!("      ({} bytes above threshold)", common - threshold);
            }
        } else {
            println!("FAIL: {} < {}", common, threshold);
            // Show divergence context
            if common < n {
                let ctx_start = common;
                let ctx_end = (common + 80).min(n);
                let ref_snip = String::from_utf8_lossy(&ref_bytes[ctx_start..ctx_end]);
                let hf2q_snip =
                    String::from_utf8_lossy(&hf2q_bytes[ctx_start..ctx_end.min(hf2q_bytes.len())]);
                println!();
                println!("Divergence at byte {}:", common);
                println!("  llama: {:?}", ref_snip);
                println!("  hf2q:  {:?}", hf2q_snip);
            }
            anyhow::bail!("Parity check failed: {} < {}", common, threshold);
        }
    }

    Ok(())
}

/// Parity capture: generate fresh hf2q output and save to reference dir.
fn cmd_parity_capture(
    model_path: &Path,
    output_dir: &Path,
    prompt_name: &str,
    max_tokens: Option<usize>,
) -> Result<()> {
    let evals_dir = Path::new("tests/evals");

    let prompts: Vec<String> = if prompt_name == "all" {
        vec![
            "sourdough".into(),
            "short_hello".into(),
            "sliding_wrap".into(),
        ]
    } else {
        vec![prompt_name.to_string()]
    };

    // Load model once
    let tokenizer_path = find_tokenizer(model_path, None)?;
    let mut ctx = gpu::GpuContext::new().map_err(|e| anyhow::anyhow!("GPU init: {e}"))?;
    let gguf = mlx_native::gguf::GgufFile::open(model_path)
        .map_err(|e| anyhow::anyhow!("GGUF open: {e}"))?;
    // ADR-022 P1.8 — prefer GGUF-self-sufficient config; see cmd_parity_check.
    let cfg = config::Gemma4Config::from_gguf(&gguf)?;
    // Model loaded once here; individual prompts re-create weights below to reset KV state
    let _gguf_preload = &gguf; // keep gguf alive
    let mut tokenizer = tokenizers::Tokenizer::from_file(&tokenizer_path)
        .map_err(|e| anyhow::anyhow!("Tokenizer: {e}"))?;
    tokenizer
        .with_truncation(None)
        .map_err(|e| anyhow::anyhow!("Tokenizer truncation: {e}"))?;

    std::fs::create_dir_all(output_dir)?;

    for pname in &prompts {
        let prompt_file = evals_dir.join("prompts").join(format!("{pname}.txt"));
        anyhow::ensure!(
            prompt_file.exists(),
            "Prompt not found: {}",
            prompt_file.display()
        );
        let prompt_text = std::fs::read_to_string(&prompt_file)?.trim().to_string();

        let tokens = max_tokens.unwrap_or(match pname.as_str() {
            "sourdough" => 1000,
            "short_hello" => 50,
            "sliding_wrap" => 500,
            _ => 200,
        });

        eprintln!("Capturing: {} ({} tokens)", pname, tokens);

        // Need to reload model for each prompt since KV cache state persists
        // Re-create model weights (reset KV caches).
        // cmd_parity has its own output contract — no progress line.
        let mut parity_progress = header::LoadProgress::new(false, 1, 0);
        let mut mlx_w_fresh = crate::inference::models::gemma4::MlxModelWeights::load_from_gguf(
            &gguf,
            &cfg,
            &mut ctx,
            &mut parity_progress,
        )?;

        let rendered = render_chat_template(
            &gguf,
            &cli::GenerateArgs {
                model: model_path.to_path_buf(),
                prompt: Some(prompt_text.clone()),
                prompt_file: None,
                tokenizer: None,
                config: None,
                temperature: 0.0,
                top_p: 1.0,
                top_k: 0,
                min_p: 0.0,
                repetition_penalty: 1.0,
                max_tokens: tokens,
                mmproj: None,
                image: None,
                chat_template: None,
                chat_template_file: None,
                benchmark: false,
                speculative: false,
                kv_bits: None,
                enable_thinking: false,
                no_thinking: false,
                ignore_eos: false,
            },
            Some(&tokenizer),
            &prompt_text,
        )?;

        let encoding = tokenizer
            .encode(rendered.as_str(), false)
            .map_err(|e| anyhow::anyhow!("Tokenize: {e}"))?;
        let prompt_tokens: Vec<u32> = encoding.get_ids().to_vec();

        let eos_token_ids: Vec<u32> = vec![1, 106];
        let first_token = mlx_w_fresh.forward_prefill(&prompt_tokens, tokens, &mut ctx)?;
        let mut all_tokens = prompt_tokens.to_vec();
        let mut next_token = first_token;
        all_tokens.push(next_token);

        for _ in 1..tokens {
            if eos_token_ids.contains(&next_token) {
                break;
            }
            let pos = all_tokens.len() - 1;
            let mut p = None;
            next_token = mlx_w_fresh.forward_decode(next_token, pos, &mut ctx, &mut p)?;
            all_tokens.push(next_token);
        }

        let gen_tokens = &all_tokens[prompt_tokens.len()..];
        let text = tokenizer.decode(gen_tokens, false).unwrap_or_default();

        let out_path = output_dir.join(format!("{pname}_hf2q.txt"));
        std::fs::write(&out_path, &text)?;
        eprintln!("  Wrote {} bytes to {}", text.len(), out_path.display());
    }

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::{
        build_chat_template_env, detect_greedy_repetition_loop,
        detect_greedy_repetition_loop_with_text, find_special_token_stop,
        llama_cpp_special_token_id_for_model, maybe_print_serve_banner, parse_scheduler_config,
        render_jinja_template, resolve_enable_thinking, run_decode_loop, should_enable_kv_persist,
        DecodeStopReason, RaisePolicy, DEFAULT_MAX_SLOTS_UNDER_INFLIGHT,
        FALLBACK_GEMMA4_API_CHAT_TEMPLATE, FALLBACK_GEMMA4_CHAT_TEMPLATE,
    };
    use crate::cli;
    use crate::core::chat_templates::QWEN3_CHATML;
    use crate::core::provenance::Provenance;
    use crate::serve::load_info::{ArchFamily, ChatTemplateSource, LoadInfo, TokenizerSource};
    use std::path::PathBuf;
    use std::time::Duration;

    #[test]
    fn llama_cpp_gpt2_special_token_defaults_match_vocab_cpp() {
        assert_eq!(
            llama_cpp_special_token_id_for_model("gpt2", "tokenizer.ggml.bos_token_id"),
            Some(11)
        );
        assert_eq!(
            llama_cpp_special_token_id_for_model("gpt2", "tokenizer.ggml.eos_token_id"),
            Some(11)
        );
        assert_eq!(
            llama_cpp_special_token_id_for_model("gpt2", "tokenizer.ggml.padding_token_id"),
            None
        );
    }

    fn synthetic_serve_banner_info() -> LoadInfo {
        LoadInfo {
            model_id: "serve-test-model".to_string(),
            arch_str: "gemma4".to_string(),
            arch_family: ArchFamily::Gemma4,
            model_path: PathBuf::from("/tmp/serve-test-model.gguf"),
            on_disk_bytes: 1024,
            backend_chip: "Apple M5 Max".to_string(),
            backend: "mlx-native",
            n_layers: 2,
            hidden_size: 32,
            vocab_size: 128,
            n_attention_heads: 4,
            n_key_value_heads: 2,
            head_dim: 8,
            sliding_window: Some(16),
            full_attention_interval: None,
            max_context_length: Some(128),
            moe: None,
            quant_label: Some("Q4_K".to_string()),
            quant_bpw: Some(4.5),
            tokenizer_source: TokenizerSource::GgufEmbedded,
            eos_token_ids: vec![1],
            bos_token_id: Some(2),
            chat_template_source: ChatTemplateSource::GgufEmbedded,
            provenance: Provenance::External,
            vision_projector: None,
            load_wall_clock: Duration::from_millis(25),
            resident_weight_bytes: None,
            kv_cache_budget_bytes: None,
            kv_spill_active: false,
            tq_kv_active: false,
            kv_bytes_per_token_override: None,
        }
    }

    #[test]
    fn cmd_serve_banner_emits_on_tty() {
        let info = synthetic_serve_banner_info();
        let mut buf = Vec::new();
        maybe_print_serve_banner(&info, &mut buf, true, false).expect("print serve banner");
        let got = String::from_utf8(buf).expect("utf8");
        // ADR-027 Phase B iter-17: banner gained `tq_kv = ...` line, so
        // the line count is now 14 (was 13).
        assert_eq!(got.lines().count(), 14);
        assert!(got.contains("hf2q load: model = serve-test-model"));
        assert!(got.contains("\x1b[2m"));
    }

    #[test]
    fn cmd_serve_banner_silent_when_non_tty() {
        let info = synthetic_serve_banner_info();
        let mut buf = Vec::new();
        maybe_print_serve_banner(&info, &mut buf, false, false).expect("skip serve banner");
        assert!(buf.is_empty());
    }

    #[test]
    fn cmd_serve_banner_silent_when_quiet() {
        let info = synthetic_serve_banner_info();
        let mut buf = Vec::new();
        maybe_print_serve_banner(&info, &mut buf, true, true).expect("skip quiet serve banner");
        assert!(buf.is_empty());
    }

    // ADR-017 §R-F1 startup-disable override regression guards.
    // Closes operator-runbook §10 gap by pinning the predicate's
    // truth table. Pure-function tests; no binary / env mutation.

    #[test]
    fn hf2q_kv_persist_zero_overrides_flag() {
        assert!(!should_enable_kv_persist(Some("/path"), Some("0")));
    }

    #[test]
    fn hf2q_kv_persist_unset_respects_flag_present() {
        assert!(should_enable_kv_persist(Some("/path"), None));
    }

    #[test]
    fn hf2q_kv_persist_one_respects_flag() {
        assert!(should_enable_kv_persist(Some("/path"), Some("1")));
    }

    #[test]
    fn hf2q_kv_persist_no_flag_means_disabled_regardless() {
        assert!(!should_enable_kv_persist(None, None));
        assert!(!should_enable_kv_persist(None, Some("0")));
        assert!(!should_enable_kv_persist(None, Some("1")));
    }

    #[test]
    fn hf2q_kv_persist_zero_with_whitespace_still_disables() {
        // Trim semantics — operators occasionally export with
        // trailing whitespace from copy-paste; `"  0  "` MUST still
        // trigger the override.
        assert!(!should_enable_kv_persist(Some("/path"), Some("  0  ")));
        assert!(!should_enable_kv_persist(Some("/path"), Some("0\n")));
    }

    #[test]
    fn hf2q_kv_persist_empty_or_other_values_respect_flag() {
        // Empty / `"true"` / `"yes"` / arbitrary strings all
        // respect the flag (only exact `"0"` after trim disables).
        assert!(should_enable_kv_persist(Some("/path"), Some("")));
        assert!(should_enable_kv_persist(Some("/path"), Some("true")));
        assert!(should_enable_kv_persist(Some("/path"), Some("yes")));
        assert!(should_enable_kv_persist(Some("/path"), Some("00")));
    }

    /// iter-219b parity-gate fix (2026-05-01) regression guard. The CLI
    /// fallback chat template MUST NOT activate Gemma 4's thinking-mode
    /// via `<|think|>` system marker — that diverged from llama.cpp's
    /// behavior and broke all 6 parity-suite checks (Gates C/D/E + F)
    /// on HEAD `3cd6ea5`. iter-217 fixed the API path's fallback the
    /// same way; this test ensures the CLI fallback stays aligned.
    ///
    /// The template MUST end with the empty `<|channel>thought\n<channel|>`
    /// block (closes thinking-mode pre-content) and MUST NOT contain
    /// `<|think|>` anywhere.
    #[test]
    fn iter219b_cli_fallback_chat_template_matches_iter217_contract() {
        assert!(
            !FALLBACK_GEMMA4_CHAT_TEMPLATE.contains("<|think|>"),
            "CLI fallback MUST NOT contain `<|think|>` (activates thinking-mode \
             that diverges from llama.cpp parity gate). Got: {FALLBACK_GEMMA4_CHAT_TEMPLATE:?}"
        );
        assert!(
            FALLBACK_GEMMA4_CHAT_TEMPLATE.contains("<|channel>thought\n<channel|>"),
            "CLI fallback MUST end with empty `<|channel>thought\\n<channel|>` block \
             (closes thinking-mode pre-content; mirrors iter-217 API-path fix). \
             Got: {FALLBACK_GEMMA4_CHAT_TEMPLATE:?}"
        );
        assert!(
            FALLBACK_GEMMA4_CHAT_TEMPLATE.contains("{{PROMPT}}"),
            "CLI fallback MUST keep `{{{{PROMPT}}}}` placeholder for `String::replace` \
             rendering. Got: {FALLBACK_GEMMA4_CHAT_TEMPLATE:?}"
        );
        // API fallback must also retain its iter-217 invariant (sibling check).
        assert!(
            !FALLBACK_GEMMA4_API_CHAT_TEMPLATE.contains("<|think|>"),
            "API fallback regression — `<|think|>` reintroduced after iter-217 fix"
        );
        assert!(
            FALLBACK_GEMMA4_API_CHAT_TEMPLATE.contains("<|channel>thought\n<channel|>"),
            "API fallback regression — empty channel block missing"
        );
    }

    /// Minimal Gemma-like template: verifies minijinja rendering of a single
    /// user message with `add_generation_prompt`.
    #[test]
    fn jinja_template_renders_single_user_turn() {
        let tmpl = "{{ bos_token }}{% for m in messages %}<|turn|>{{ m.role }}\n{{ m.content }}<|end|>\n{% endfor %}{% if add_generation_prompt %}<|turn|>model\n{% endif %}";
        let out = render_jinja_template(tmpl, "hello", None).expect("render ok");
        assert!(
            out.starts_with("<bos>"),
            "output should start with bos_token: {out}"
        );
        assert!(
            out.contains("<|turn|>user\nhello<|end|>"),
            "user turn missing: {out}"
        );
        assert!(
            out.ends_with("<|turn|>model\n"),
            "generation prompt missing: {out}"
        );
    }

    /// Parse failure on an invalid Jinja template should surface as an error.
    #[test]
    fn jinja_template_parse_error_is_reported() {
        let tmpl = "{% unclosed"; // invalid
        let err = render_jinja_template(tmpl, "x", None).unwrap_err();
        let msg = format!("{err:#}");
        assert!(
            msg.contains("parse") || msg.contains("Jinja") || msg.contains("template"),
            "expected parse error, got: {msg}"
        );
    }

    // ── ADR-005 Phase 4 reopen iter-215 Wedge-2 — load_engine arch dispatch ──
    //
    // Iter-215 replaces the iter-214 wedge-1 bail (commit 064797e) with
    // actual SERVE-side dispatch.  The pre-iter-215 tests
    // (load_engine_rejects_qwen35_arch_with_operator_actionable_error,
    //  load_engine_rejects_qwen35moe_arch_with_operator_actionable_error,
    //  load_engine_does_not_reject_unknown_arch_at_wedge)
    // tested the bail behavior; they are replaced below by tests that
    // assert the new dispatch routes to the correct LoadedModel variant.

    /// Write a minimal valid GGUF (magic + version + 0 tensors + N
    /// string KVs) to a tempfile. Just enough structure for the
    /// header-only parse to succeed and `metadata_string` to find
    /// the keys.
    fn write_minimal_gguf_with_arch(arch: &str) -> tempfile::NamedTempFile {
        use std::io::Write;
        let mut f = tempfile::Builder::new()
            .suffix(".gguf")
            .tempfile()
            .expect("tempfile");
        let mut buf: Vec<u8> = Vec::new();
        buf.extend_from_slice(b"GGUF");
        buf.extend_from_slice(&3u32.to_le_bytes()); // version
        buf.extend_from_slice(&0u64.to_le_bytes()); // tensor_count
        buf.extend_from_slice(&1u64.to_le_bytes()); // metadata_kv_count = 1
                                                    // KV: key="general.architecture" value=<arch>
        let key = b"general.architecture";
        buf.extend_from_slice(&(key.len() as u64).to_le_bytes());
        buf.extend_from_slice(key);
        buf.extend_from_slice(&8u32.to_le_bytes()); // GGUF_TYPE_STRING = 8
        let val = arch.as_bytes();
        buf.extend_from_slice(&(val.len() as u64).to_le_bytes());
        buf.extend_from_slice(val);
        f.write_all(&buf).expect("write");
        f.flush().expect("flush");
        f
    }

    /// Synthetic GGUF with arch=qwen35 routes through
    /// `Qwen35LoadedModel::load`.  The minimal GGUF has 0 tensors so
    /// the full weights load fails downstream — but the failure
    /// surface MUST be from the qwen35 path, NOT the pre-iter-215
    /// bail or the Gemma loader's `attn_q.weight not found` panic.
    #[test]
    fn load_engine_routes_qwen35_to_qwen35_loaded_model() {
        let tmp = write_minimal_gguf_with_arch("qwen35");
        let cfg = super::multi_model::EngineConfig {
            tokenizer_path: None,
            config_path: None,
            queue_capacity: 4,
            warmup_synchronously: false,
            kv_metrics_sink: None,
            dwq_overlay_path: None,
            // ADR-040 Phase C iter-4 (C4) — test path stays on the
            // SerialFifo default (the ADR-005 byte-equivalent route).
            engine_mode: crate::serve::api::engine::EngineMode::SerialFifo,
        };
        let result = super::load_engine(tmp.path(), &cfg);
        // 0-tensor GGUF can't fully load, but the FAILURE shape proves
        // dispatch reached the qwen35 path.
        assert!(result.is_err(), "0-tensor synthetic GGUF must fail load");
        let msg = format!("{:#}", result.err().unwrap());
        // Wedge-1 bail message is GONE.
        assert!(
            !msg.contains("Phase 4 follow-up: SERVE-side load for general.architecture"),
            "wedge-1 bail must not fire post iter-215; got: {msg}"
        );
        // Wedge-1 message body referenced cmd_generate_qwen35 — the
        // SERVE-side load must NOT name that as a workaround anymore
        // (the SERVE path handles qwen35 now; chat returns 501 from
        // the worker arm, not load_engine).  The downstream
        // Qwen35Model::load_from_gguf failure does not name it.
        assert!(
            !msg.contains("hf2q generate") || !msg.contains("cmd_generate_qwen35"),
            "load_engine error must not include the iter-214 wedge-1 workaround \
             pointer; got: {msg}"
        );
    }

    /// Symmetric for qwen35moe (MoE variant routes through the same
    /// arm because the dispatcher's match covers both).
    #[test]
    fn load_engine_routes_qwen35moe_to_qwen35_loaded_model() {
        let tmp = write_minimal_gguf_with_arch("qwen35moe");
        let cfg = super::multi_model::EngineConfig {
            tokenizer_path: None,
            config_path: None,
            queue_capacity: 4,
            warmup_synchronously: false,
            kv_metrics_sink: None,
            dwq_overlay_path: None,
            // ADR-040 Phase C iter-4 (C4) — test path stays on the
            // SerialFifo default (the ADR-005 byte-equivalent route).
            engine_mode: crate::serve::api::engine::EngineMode::SerialFifo,
        };
        let result = super::load_engine(tmp.path(), &cfg);
        assert!(result.is_err(), "0-tensor synthetic GGUF must fail load");
        let msg = format!("{:#}", result.err().unwrap());
        assert!(
            !msg.contains("Phase 4 follow-up: SERVE-side load for general.architecture"),
            "wedge-1 bail must not fire post iter-215; got: {msg}"
        );
    }

    // ── Wedge-4 / iter-227 — Qwen3-VL arch dispatch tests ──
    //
    // Pin the new dispatch arm in `LoadedModel::load` (api/engine.rs)
    // and `cmd_generate` (this file's runtime arch peek) so the
    // operator-actionable error message reaches the caller instead of
    // the pre-iter-227 silent fall-through into the Gemma 4 loader's
    // `missing blk.0.ffn_gate_up_exps.weight` panic.

    /// Synthetic GGUF with arch=qwen3_vl (hf2q's underscored convention,
    /// emitted by Wedge-4f convert) used to surface the iter-227
    /// dispatch bail. As of **iter-228a** (2026-05-02) the dense
    /// Qwen3-VL arm no longer bails at the dispatch site — it routes
    /// through `Qwen3VlTextLoadedModel::load`, which then fails
    /// downstream when the synthetic 0-tensor GGUF has no
    /// `qwen3_vl.block_count` etc. metadata to parse a config from.
    /// This test pins the new failure shape: dispatch reaches the
    /// iter-228a config parser, NOT the Gemma loader (the regression
    /// guard the iter-227 test was originally for is preserved).
    #[test]
    fn iter228a_load_engine_routes_qwen3_vl_to_qwen3vl_text_loader() {
        let tmp = write_minimal_gguf_with_arch("qwen3_vl");
        let cfg = super::multi_model::EngineConfig {
            tokenizer_path: None,
            config_path: None,
            queue_capacity: 4,
            warmup_synchronously: false,
            kv_metrics_sink: None,
            dwq_overlay_path: None,
            // ADR-040 Phase C iter-4 (C4) — test path stays on the
            // SerialFifo default (the ADR-005 byte-equivalent route).
            engine_mode: crate::serve::api::engine::EngineMode::SerialFifo,
        };
        let result = super::load_engine(tmp.path(), &cfg);
        assert!(
            result.is_err(),
            "0-tensor synthetic qwen3_vl GGUF must fail load (no metadata)"
        );
        let msg = format!("{:#}", result.err().unwrap());
        // Must reach the Qwen3VlTextConfig parser (not Gemma).
        assert!(
            msg.contains("Qwen3VlTextConfig::from_gguf")
                || msg.contains("missing core architecture facts"),
            "iter-228a must route dense Qwen3-VL through Qwen3VlTextConfig parser; got: {msg}"
        );
        // Regression guards from the iter-227 tests:
        // (1) MUST NOT fall through to the Gemma loader's panic surface.
        assert!(
            !msg.contains("missing blk.0.ffn_gate_up_exps.weight"),
            "iter-228a must intercept BEFORE the Gemma MoE expert load; got: {msg}"
        );
        // (2) MUST NOT regress to the iter-227 actionable-error bail
        // for dense Qwen3-VL (that bail is now lifted; only MoE bails).
        assert!(
            !msg.contains("iter-227 closes only the dispatch gap"),
            "iter-228a lifted the iter-227 dispatch bail for dense Qwen3-VL; got: {msg}"
        );
    }

    /// Symmetric for the upstream llama.cpp arch-string convention
    /// (no underscore — `qwen3vl`). Both arch spellings route through
    /// the iter-228a load path.
    #[test]
    fn iter228a_load_engine_routes_qwen3vl_upstream_to_qwen3vl_text_loader() {
        let tmp = write_minimal_gguf_with_arch("qwen3vl");
        let cfg = super::multi_model::EngineConfig {
            tokenizer_path: None,
            config_path: None,
            queue_capacity: 4,
            warmup_synchronously: false,
            kv_metrics_sink: None,
            dwq_overlay_path: None,
            // ADR-040 Phase C iter-4 (C4) — test path stays on the
            // SerialFifo default (the ADR-005 byte-equivalent route).
            engine_mode: crate::serve::api::engine::EngineMode::SerialFifo,
        };
        let result = super::load_engine(tmp.path(), &cfg);
        assert!(result.is_err());
        let msg = format!("{:#}", result.err().unwrap());
        assert!(
            msg.contains("Qwen3VlTextConfig::from_gguf")
                || msg.contains("missing core architecture facts"),
            "iter-228a must route upstream-arch dense Qwen3-VL through Qwen3VlTextConfig parser; got: {msg}"
        );
    }

    /// MoE Qwen3-VL variant (Qwen3-VL-30B-A3B etc., upstream
    /// `qwen3vlmoe` arch). The error must distinguish MoE from dense
    /// so the operator knows which variant we recognized.
    #[test]
    fn iter227_load_engine_rejects_qwen3vlmoe_with_moe_specific_error() {
        let tmp = write_minimal_gguf_with_arch("qwen3vlmoe");
        let cfg = super::multi_model::EngineConfig {
            tokenizer_path: None,
            config_path: None,
            queue_capacity: 4,
            warmup_synchronously: false,
            kv_metrics_sink: None,
            dwq_overlay_path: None,
            // ADR-040 Phase C iter-4 (C4) — test path stays on the
            // SerialFifo default (the ADR-005 byte-equivalent route).
            engine_mode: crate::serve::api::engine::EngineMode::SerialFifo,
        };
        let result = super::load_engine(tmp.path(), &cfg);
        assert!(result.is_err());
        let msg = format!("{:#}", result.err().unwrap());
        assert!(
            msg.contains("Qwen3-VL") && msg.contains("MoE"),
            "iter-227 MoE-variant error must include 'MoE' to distinguish from dense; got: {msg}"
        );
    }

    /// Unknown architectures must fail closed at dispatch rather than
    /// being guessed to be Gemma.  A wrong architectural fallback can
    /// otherwise turn a metadata error into a misleading tensor error.
    #[test]
    fn load_engine_rejects_unknown_arch_without_gemma_fallback() {
        let tmp = write_minimal_gguf_with_arch("totally-fake-arch-name");
        let cfg = super::multi_model::EngineConfig {
            tokenizer_path: None,
            config_path: None,
            queue_capacity: 4,
            warmup_synchronously: false,
            kv_metrics_sink: None,
            dwq_overlay_path: None,
            // ADR-040 Phase C iter-4 (C4) — test path stays on the
            // SerialFifo default (the ADR-005 byte-equivalent route).
            engine_mode: crate::serve::api::engine::EngineMode::SerialFifo,
        };
        let result = super::load_engine(tmp.path(), &cfg);
        assert!(result.is_err(), "unknown architecture must fail dispatch");
        let msg = format!("{:#}", result.err().unwrap());
        assert!(
            msg.contains("unsupported GGUF general.architecture")
                && msg.contains("totally-fake-arch-name"),
            "unknown architecture error must identify the rejected value; got: {msg}"
        );
        assert!(
            !msg.contains("missing blk.0.ffn_gate_up_exps.weight"),
            "unknown architecture must not reach the Gemma loader; got: {msg}"
        );
    }

    /// DeepSeek-V4 is routed into its dedicated native loader. A deliberately
    /// incomplete GGUF must fail at DeepSeek validation, never at Gemma
    /// tensor lookup or the retired "runtime unavailable" dispatch guard.
    #[test]
    fn load_engine_routes_deepseek4_to_native_loader_without_gemma_fallback() {
        let tmp = write_minimal_gguf_with_arch("deepseek4");
        let cfg = super::multi_model::EngineConfig {
            tokenizer_path: None,
            config_path: None,
            queue_capacity: 4,
            warmup_synchronously: false,
            kv_metrics_sink: None,
            dwq_overlay_path: None,
            engine_mode: crate::serve::api::engine::EngineMode::SerialFifo,
        };
        let result = super::load_engine(tmp.path(), &cfg);
        assert!(
            result.is_err(),
            "the minimal DeepSeek-V4 fixture must fail validation"
        );
        let msg = format!("{:#}", result.err().unwrap());
        assert!(
            msg.contains("DeepSeek-V4 tokenizer") || msg.contains("native DeepSeek-V4 model"),
            "DeepSeek-V4 must reach its native loader; got: {msg}"
        );
        assert!(
            !msg.contains("refusing to route it through Gemma")
                && !msg.contains("missing blk.0.ffn_gate_up_exps.weight"),
            "DeepSeek-V4 must not reach the Gemma loader; got: {msg}"
        );
    }

    /// Iter-227 regression guard for the existing qwen35 / qwen35moe
    /// dispatch arms — they MUST still route to `Qwen35LoadedModel::load`,
    /// NOT into the new iter-227 Qwen3-VL bail. The new is_qwen3_vl_arch
    /// predicate is narrow and additive.
    #[test]
    fn iter227_does_not_regress_qwen35_dispatch() {
        for arch in &["qwen35", "qwen35moe"] {
            let tmp = write_minimal_gguf_with_arch(arch);
            let cfg = super::multi_model::EngineConfig {
                tokenizer_path: None,
                config_path: None,
                queue_capacity: 4,
                warmup_synchronously: false,
                kv_metrics_sink: None,
                dwq_overlay_path: None,
                // ADR-040 Phase C iter-4 (C4) — test path stays on the
                // SerialFifo default (the ADR-005 byte-equivalent route).
                engine_mode: crate::serve::api::engine::EngineMode::SerialFifo,
            };
            let result = super::load_engine(tmp.path(), &cfg);
            assert!(
                result.is_err(),
                "0-tensor synthetic GGUF for {arch} must fail load (routed to qwen35 path)"
            );
            let msg = format!("{:#}", result.err().unwrap());
            // The Qwen3-VL bail message must NOT fire on real qwen35
            // arch strings — that would mean iter-227's predicate
            // widened past its intended scope.
            assert!(
                !msg.contains("Qwen3-VL"),
                "iter-227 Qwen3-VL dispatch must NOT fire on arch={arch}; got: {msg}"
            );
        }
    }

    // ---- Greedy repetition-loop detector tests ----
    //
    // 2026-05-02 user reported the qwen35 generate CLI looping for 1014
    // tokens of "I should make sure the response is accurate and up-to-date."
    // before `<|im_end|>` finally fired. Pure greedy on thinking-style
    // checkpoints is brittle. The detector below is the deterministic-mode
    // escape hatch — sampling (temp/top_k/top_p) lands as a follow-up.

    #[test]
    fn detect_repetition_returns_none_below_window_size() {
        let toks: Vec<u32> = (0..50).collect();
        assert_eq!(detect_greedy_repetition_loop(&toks), None);
    }

    #[test]
    fn detect_repetition_returns_none_for_diverse_tokens() {
        let toks: Vec<u32> = (0..200).collect();
        assert_eq!(detect_greedy_repetition_loop(&toks), None);
    }

    #[test]
    fn detect_repetition_finds_8_token_cycle() {
        // 8-token cycle repeated 16 times → 128-token window full of one cycle.
        let cycle: [u32; 8] = [101, 102, 103, 104, 105, 106, 107, 108];
        let mut toks: Vec<u32> = (0..50).collect();
        for _ in 0..20 {
            toks.extend_from_slice(&cycle);
        }
        let result = detect_greedy_repetition_loop(&toks);
        assert!(result.is_some(), "8-token cycle should be detected");
        let (ngram, occ) = result.unwrap();
        // Detector now returns the SMALLEST matching n.  An 8-cycle of
        // [101..108] has no smaller subcycle (no repeats within 8 unique
        // tokens), so ngram=8 is the smallest match.  Required-occurrence
        // floor relaxed from 4 to 3 (2026-05-16 bulletproof rewrite).
        assert_eq!(ngram, 8, "should detect at the smallest matching size");
        assert!(
            occ >= 3,
            "should report ≥3 occurrences in a saturated window, got {occ}"
        );
    }

    // ---- Bulletproof-rewrite coverage (2026-05-16) ----
    //
    // Each test below is a cycle size that the prior fixed-set detector
    // (sizes {8,12,16,20,24}) SILENTLY MISSED.  Now they must all fire.

    #[test]
    fn detect_repetition_finds_2_token_cycle_single_token_loop() {
        // "AAAAAA" — model stuck on one token (technically a 2-cycle of
        // [A,A] but covers single-token degenerate loops in practice).
        let mut toks: Vec<u32> = (0..30).collect();
        for _ in 0..10 {
            toks.push(777);
        }
        let result = detect_greedy_repetition_loop(&toks);
        assert!(
            result.is_some(),
            "single-token loop should be detected (was missed by prior detector)"
        );
        let (ngram, _) = result.unwrap();
        assert_eq!(ngram, 2, "should detect at the smallest matching size");
    }

    #[test]
    fn detect_repetition_finds_7_token_cycle() {
        // 7-token cycle — NOT in the prior fixed set {8,12,16,20,24}.
        let cycle: [u32; 7] = [501, 502, 503, 504, 505, 506, 507];
        let mut toks: Vec<u32> = (0..30).collect();
        for _ in 0..10 {
            toks.extend_from_slice(&cycle);
        }
        let result = detect_greedy_repetition_loop(&toks);
        assert!(
            result.is_some(),
            "7-token cycle should be detected (was missed by prior detector)"
        );
        assert_eq!(result.unwrap().0, 7);
    }

    #[test]
    fn detect_repetition_finds_11_token_cycle() {
        // 11-token cycle — NOT in the prior fixed set.
        let cycle: [u32; 11] = [601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611];
        let mut toks: Vec<u32> = (0..30).collect();
        for _ in 0..10 {
            toks.extend_from_slice(&cycle);
        }
        let result = detect_greedy_repetition_loop(&toks);
        assert!(
            result.is_some(),
            "11-token cycle should be detected (was missed by prior detector)"
        );
        assert_eq!(result.unwrap().0, 11);
    }

    #[test]
    fn detect_repetition_finds_18_token_cycle() {
        // 18-token cycle — between the prior fixed-set steps {16, 20}.
        // Approximates the "Miticidal/Fungicidal/Herbicidal/Insecticidal
        // Agent" failure mode from the 2026-05-16 user report (4 terms
        // each ~4-5 BPE tokens plus delimiters → cycle ~18 tokens).
        let cycle: [u32; 18] = [
            701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717,
            718,
        ];
        let mut toks: Vec<u32> = (0..30).collect();
        for _ in 0..8 {
            toks.extend_from_slice(&cycle);
        }
        let result = detect_greedy_repetition_loop(&toks);
        assert!(
            result.is_some(),
            "18-token cycle should be detected (was missed by prior detector)"
        );
        assert_eq!(result.unwrap().0, 18);
    }

    #[test]
    fn detect_repetition_finds_22_token_cycle() {
        // 22-token cycle — between the prior fixed-set steps {20, 24}.
        let cycle: [u32; 22] = [
            801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817,
            818, 819, 820, 821, 822,
        ];
        let mut toks: Vec<u32> = (0..30).collect();
        for _ in 0..6 {
            toks.extend_from_slice(&cycle);
        }
        let result = detect_greedy_repetition_loop(&toks);
        assert!(
            result.is_some(),
            "22-token cycle should be detected (was missed by prior detector)"
        );
        assert_eq!(result.unwrap().0, 22);
    }

    #[test]
    fn detect_repetition_no_false_positive_on_two_repeats() {
        // Bigger threshold check: a pattern that REPEATS ONLY TWICE is
        // not a loop (could be legitimate parallelism in prose).  The
        // detector requires ≥3 consecutive occurrences before firing.
        let cycle: [u32; 8] = [901, 902, 903, 904, 905, 906, 907, 908];
        let mut toks: Vec<u32> = (0..30).collect();
        for _ in 0..2 {
            toks.extend_from_slice(&cycle);
        }
        assert_eq!(detect_greedy_repetition_loop(&toks), None);
    }

    #[test]
    fn detect_repetition_finds_16_token_cycle() {
        // 16-token cycle won't be detected at ngram=8 (key not periodic at
        // smaller granularity for arbitrary tokens), but at ngram=16 should
        // fire.
        let cycle: [u32; 16] = [
            201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216,
        ];
        let mut toks: Vec<u32> = (0..50).collect();
        for _ in 0..10 {
            toks.extend_from_slice(&cycle);
        }
        let result = detect_greedy_repetition_loop(&toks);
        assert!(result.is_some(), "16-token cycle should be detected");
    }

    #[test]
    fn detect_repetition_smart_skips_structural_cycle_at_3_reps() {
        // Markdown table separator: cycle decodes to "| :--- " (all
        // punctuation/whitespace).  At MIN_OCCURRENCES_CONTENT=3 the
        // basic detector would bail; the smart detector must NOT bail
        // when alpha_ratio < 0.30 — it should require 8+ reps instead.
        //
        // Real-world failure mode observed on probe_long after the
        // batched-prefill workaround restored coherence: model genuinely
        // emits a 3-column markdown separator (3 reps) and was caught
        // as a false positive by the basic detector.
        let cycle: [u32; 4] = [101, 102, 103, 104];
        let mut toks: Vec<u32> = (0..30).collect();
        for _ in 0..3 {
            toks.extend_from_slice(&cycle);
        }
        let result = detect_greedy_repetition_loop_with_text(&toks, |_cycle| {
            // Simulate tokenizer.decode → "| :--- " (no letters).
            "| :--- ".to_string()
        });
        assert!(
            result.is_none(),
            "structural cycle (alpha_ratio=0) at only 3 reps should NOT fire"
        );

        // 8+ reps should still fire — at that point even a structural
        // cycle is suspicious.
        let mut toks2: Vec<u32> = (0..30).collect();
        for _ in 0..8 {
            toks2.extend_from_slice(&cycle);
        }
        let result2 =
            detect_greedy_repetition_loop_with_text(&toks2, |_cycle| "| :--- ".to_string());
        assert!(
            result2.is_some(),
            "structural cycle at 8 reps should fire (genuine pad-loop)"
        );
        let (_ng, reps) = result2.unwrap();
        assert!(reps >= 8, "structural threshold should be ≥ 8 reps");
    }

    #[test]
    fn detect_repetition_smart_fires_on_content_cycle_at_3_reps() {
        // Real content cycle ("Miticidal Agent, Fungicidal Agent, ..."):
        // alpha-rich, MUST fire at 3 reps.
        let cycle: [u32; 5] = [201, 202, 203, 204, 205];
        let mut toks: Vec<u32> = (0..30).collect();
        for _ in 0..3 {
            toks.extend_from_slice(&cycle);
        }
        let result = detect_greedy_repetition_loop_with_text(&toks, |_cycle| {
            "Miticidal Agent, ".to_string()
        });
        assert!(
            result.is_some(),
            "alphanumeric content cycle at 3 reps MUST fire"
        );
        let (_ng, reps) = result.unwrap();
        assert_eq!(reps, 3, "content threshold should be 3 reps");
    }

    #[test]
    fn detect_repetition_tolerates_minor_drift() {
        // User's actual case: cycle is mostly identical with one outlier
        // mid-stream. Build 128-token window where last 16 tokens repeat
        // 5+ times non-consecutively (one outlier replaces a single token
        // in the middle).
        let cycle: [u32; 12] = [301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312];
        let outlier: u32 = 999;
        let mut toks: Vec<u32> = (0..30).collect();
        for i in 0..10 {
            if i == 4 {
                // Outlier replaces one token mid-cycle.
                let mut drifted = cycle.to_vec();
                drifted[6] = outlier;
                toks.extend_from_slice(&drifted);
            } else {
                toks.extend_from_slice(&cycle);
            }
        }
        let result = detect_greedy_repetition_loop(&toks);
        assert!(
            result.is_some(),
            "minor-drift cycle should still detect (non-consecutive matching), got None"
        );
    }

    #[test]
    fn detect_repetition_does_not_overcount_with_overlapping_increment() {
        // Pin the non-overlap invariant: occurrences MUST count non-overlapping
        // blocks of `ngram` tokens. For 200 zeros at ngram=2 the truthful count
        // is exactly 100 blocks (not 199 overlapping bigrams). The pre-9f797761
        // detector used a 128-token window-scan that could over-count short
        // n-grams; the current consecutive-only walk (in `ngram`-sized steps)
        // is non-overlapping by construction.
        let toks: Vec<u32> = vec![0; 200];
        let result = detect_greedy_repetition_loop(&toks);
        assert!(result.is_some());
        let (ngram, occ) = result.unwrap();
        assert!(
            occ.saturating_mul(ngram) <= toks.len(),
            "non-overlap invariant violated: ngram={ngram} occ={occ}, occ*ngram={} > n={}",
            occ.saturating_mul(ngram),
            toks.len()
        );
    }

    #[test]
    fn detect_repetition_does_not_fire_on_bullet_list() {
        // 2026-05-03 — regression pin for the wedding-cake false-positive.
        // A shopping list with 6 bulleted items shares the prefix
        //   `*   **Buy:**` (~8 tokens)
        // followed by completely different content per bullet. The OLD
        // window-scan detector (any 4 non-overlapping matches of the
        // last-N-token key in the last 128 tokens) fired here because
        // the prefix tokens incidentally appeared 4× in the window —
        // even though the model was making real progress. Confirm the
        // consecutive-only detector ignores this.
        let bullet_prefix: [u32; 8] = [42, 32, 32, 32, 1234, 5678, 9012, 3456];
        // synthetic non-prefix bytes per bullet, growing each time
        let mut toks: Vec<u32> = (0..50).collect();
        for content_seed in 0..6u32 {
            toks.extend_from_slice(&bullet_prefix);
            // 12-20 tokens of unique content per bullet
            for j in 0..(12 + content_seed as usize) {
                toks.push(50000 + content_seed * 100 + j as u32);
            }
        }
        // The last 8 tokens are NOT all the prefix — the trailing content
        // is unique. detect_greedy_repetition_loop should NOT fire.
        assert_eq!(
            detect_greedy_repetition_loop(&toks),
            None,
            "false positive on legitimate bullet-list enumeration"
        );
    }

    #[test]
    fn detect_repetition_fires_on_true_consecutive_loop() {
        // 2026-05-03 — pin the true-loop detection as the inverse of the
        // bullet-list test. The 2026-05-02 French-Toast user report had
        // the model emitting `"I should make sure the response is
        // accurate and up-to-date."` × 30+ in a row. That sequence,
        // tokenized, is roughly 16 tokens; consecutive emission for 30
        // cycles trivially satisfies "last 4 × 16 tokens are 4 copies
        // of the last 16 tokens". Verify the new detector still fires.
        let cycle: [u32; 16] = [
            701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716,
        ];
        let mut toks: Vec<u32> = (0..50).collect();
        for _ in 0..30 {
            toks.extend_from_slice(&cycle);
        }
        let result = detect_greedy_repetition_loop(&toks);
        assert!(
            result.is_some(),
            "true consecutive loop must still trigger the detector"
        );
        let (ngram, _occ) = result.unwrap();
        assert_eq!(
            ngram, 16,
            "16-token cycle's halves are NOT individually periodic at 8, so \
             ngram=8 doesn't fire; the smallest matching consecutive period \
             is 16"
        );
    }

    // ---- find_special_token_stop matrix --------------------------------
    //
    // 2026-05-02 — pin the substring stop policy with a falsifiable test
    // matrix so it can't drift silently the way the inline `const` did
    // (three commits, opposite directions, all uncovered). Each test case
    // names a real model-output pattern observed in user reports OR a
    // hypothetical we want to NOT trip on.

    #[test]
    fn special_token_stop_empty_input() {
        assert_eq!(find_special_token_stop(""), None);
    }

    #[test]
    fn special_token_stop_plain_thinking_text_does_not_stop() {
        // Bare thinking-style content with no markers — must continue.
        let s = "I should make sure the answer is comprehensive and avoids \
                 jargon. The user is asking about candy.";
        assert_eq!(find_special_token_stop(s), None);
    }

    #[test]
    fn special_token_stop_think_html_tag_does_not_stop() {
        // `</think>` end-of-thinking tag is NOT in the stop set — it's a
        // legitimate thinking-mode transition; the model continues to the
        // actual answer afterwards.
        let s = "<think>reasoning content</think>\n\nactual answer";
        assert_eq!(find_special_token_stop(s), None);
    }

    #[test]
    fn special_token_stop_im_end_alone_is_not_a_stop() {
        // 2026-05-03 — `<|im_end|>` was REMOVED from SPECIAL_TOKEN_STOPS.
        // On thinking-capable Qwen3 finetunes (e.g. abliterix-EGA-
        // abliterated dwq48) the model emits `<|im_end|>` as a SEGMENT
        // separator BETWEEN thinking and the actual answer, NOT as a
        // turn-end marker. Stopping there cut generation off before the
        // answer was emitted. The turn-end signal on this finetune is the
        // NEXT-turn opener `<|im_start|>user` — see
        // `special_token_stop_im_start_user_signals_turn_end` below.
        let s = "answer text<|im_end|>";
        assert_eq!(find_special_token_stop(s), None);
    }

    #[test]
    fn special_token_stop_im_start_user_signals_turn_end() {
        // The model hallucinating the next user turn = it has finished
        // its assistant turn. Stop here.
        let s = "[thinking]<|im_end|>\n<|im_start|>assistant\n[answer]<|im_end|>\n<|im_start|>user\nnext";
        assert_eq!(find_special_token_stop(s), Some("<|im_start|>user"));
    }

    #[test]
    fn special_token_stop_endoftext_fragment_stops() {
        let s = "answer text<|endoftext|>";
        assert_eq!(find_special_token_stop(s), Some("<|endoftext|>"));
    }

    #[test]
    fn special_token_stop_end_marker_before_im_start_stops_at_end() {
        // 2026-05-02 candy-thinking degeneracy: post-`<|end|>` model
        // hallucinates a new `<|im_start|>user` turn and echoes the prompt.
        // `<|im_start|>` is not a llama.cpp-style stop by itself, but the
        // preceding end marker is.
        let s = "thinking content<|end|>\n\n<|im_start|>user\necho";
        assert_eq!(find_special_token_stop(s), Some("<|end|>"));
    }

    #[test]
    fn special_token_stop_im_end_then_assistant_role_no_user_does_not_stop() {
        // 2026-05-03 — the original French-Toast leak fixture
        // (`<|im_end|>` followed by `<|im_start|>assistant`) is NOT a
        // turn boundary on the finetunes that emit `<|im_end|>` as a
        // segment separator. Without a `<|im_start|>user` follow-up,
        // generation continues. (For models that emit id 151645
        // properly, integer-EOS catches first; this list is the
        // text-fragment fallback only.)
        let s = "answer<|im_end|>\n<|im_start|>assistant\nleak";
        assert_eq!(find_special_token_stop(s), None);
    }

    #[test]
    fn special_token_stop_phi3_end_marker_stops() {
        // Phi-3-style end-of-turn marker emitted as multi-token text on a
        // Qwen3.x-architecture GGUF (observed 2026-05-02 candy prompt).
        let s = "answer<|end|>";
        assert_eq!(find_special_token_stop(s), Some("<|end|>"));
    }

    #[test]
    fn special_token_stop_end_substring_does_not_falsely_match_endoftext() {
        // Sanity: `<|end|>` (7 chars: `<|end|>`) is NOT a substring of
        // `<|endoftext|>` (after `<|end` we have `oftext|>`, not `|>`).
        // When only `<|endoftext|>` is present, the returned marker must be
        // `<|endoftext|>`.
        let s = "answer<|endoftext|>";
        assert_eq!(find_special_token_stop(s), Some("<|endoftext|>"));
    }

    #[test]
    fn special_token_stop_degenerate_im_start_only_does_not_stop() {
        // llama.cpp does not treat `<|im_start|>` as an implicit reverse
        // prompt in the normal completion sampler. Stopping here causes the
        // Qwen no-thinking path to terminate before the model can continue
        // after a generated ChatML opener.
        let s = "<|im_start|>\n\n<|im_start|>";
        assert_eq!(find_special_token_stop(s), None);
    }

    #[test]
    fn special_token_stop_qwen3_thinking_clean_answer_does_not_stop_mid_text() {
        // Negative case: realistic Qwen3-thinking output without any
        // turn-leak markers must NOT stop mid-text.
        let s = "<think>The user wants candy.</think>\n\n\
                 To make candy you need sugar, water, and flavoring. \
                 Heat to soft-ball stage at 235°F.";
        assert_eq!(find_special_token_stop(s), None);
    }

    #[test]
    fn special_token_stop_earliest_marker_wins() {
        // Pin the tie-break: when multiple stops are present, stop at the
        // earliest byte position. This matches streaming behavior: once a
        // complete stop marker appears, later text is not part of the answer.
        let s = "x<|end|>y<|im_end|>z";
        assert_eq!(find_special_token_stop(s), Some("<|end|>"));
    }

    // Removed 2026-05-03: `special_token_safe_prefix` and
    // `qwen35_visible_generated_prefix` — the inference-layer visible-text
    // surgery they implemented was the brittle scaffolding the user called
    // out. run_decode_loop now emits raw decoded tokens; only the 3-string
    // `SPECIAL_TOKEN_STOPS` list remains (matches llama.cpp's stop-string
    // contract). The two helpers + their tests are removed alongside.

    // ---- enable_thinking plumbing into chat-template render ----------
    //
    // 2026-05-02 — pin the H2-audit fix (docs/research/decode-test-gap-2026-05-02.md).
    // `render_jinja_template` was not passing `enable_thinking` into the
    // Jinja context; the qwen3-chatml.jinja `else` branch at line 152 always
    // fired, opening an unfilled `<think>\n` block. On non-thinking
    // checkpoints the model improvised a close marker (Phi-3's `<|end|>` in
    // the user's candy-prompt repro 2026-05-02). These tests pin the three
    // resolved states (Some(true), Some(false), None) against the bundled
    // qwen3-chatml.jinja vendor fixture so a future regression on the plumb
    // surface fails at PR time, not at the operator's terminal.

    #[test]
    fn render_jinja_qwen3_chatml_enable_thinking_true_opens_unfilled_think_block() {
        let rendered = render_jinja_template(QWEN3_CHATML, "make me candy", Some(true))
            .expect("qwen3-chatml render with enable_thinking=true must succeed");
        assert!(
            rendered.ends_with("<|im_start|>assistant\n<think>\n"),
            "with enable_thinking=true the qwen3-chatml `else` branch (line 152) \
             must fire, ending the prompt with `<|im_start|>assistant\\n<think>\\n` \
             so the model emits reasoning content next. Actual tail: \
             `{}`",
            &rendered[rendered.len().saturating_sub(60)..]
        );
    }

    #[test]
    fn render_jinja_qwen3_chatml_enable_thinking_false_emits_pre_closed_think_block() {
        let rendered = render_jinja_template(QWEN3_CHATML, "make me candy", Some(false))
            .expect("qwen3-chatml render with enable_thinking=false must succeed");
        assert!(
            rendered.ends_with("<|im_start|>assistant\n<think>\n\n</think>\n\n"),
            "with enable_thinking=false the qwen3-chatml `if` branch (line 150) \
             must fire, emitting a pre-closed `<think>\\n\\n</think>\\n\\n` block \
             so the model skips reasoning and emits the answer directly. Actual \
             tail: `{}`",
            &rendered[rendered.len().saturating_sub(60)..]
        );
    }

    #[test]
    fn render_jinja_qwen3_chatml_enable_thinking_none_treats_as_undefined_else_branch() {
        // Property pin: when callers pass None to render_jinja_template
        // directly (e.g. an internal API path that hasn't been updated to
        // resolve the default), Jinja sees the variable as undefined. The
        // `enable_thinking is defined and ... is false` guard fails; the
        // `else` branch fires; output matches Some(true).
        //
        // NOTE: `render_chat_template` no longer EVER passes None — the
        // 2026-05-02 user-regression follow-up changed the default
        // resolution to Some(false). This test pins the LOWER-LEVEL helper
        // semantics (None == undefined Jinja var) so future callers know
        // what the contract is, even though the upper-level CLI path
        // doesn't exercise None anymore.
        let rendered_none = render_jinja_template(QWEN3_CHATML, "make me candy", None)
            .expect("qwen3-chatml render with None must succeed");
        let rendered_true = render_jinja_template(QWEN3_CHATML, "make me candy", Some(true))
            .expect("qwen3-chatml render with Some(true) must succeed");
        assert_eq!(
            rendered_none, rendered_true,
            "with the qwen3-chatml template, None must render byte-identically \
             to Some(true) — the `is defined and is false` guard fails for both. \
             A divergence here means minijinja serialized None as a non-undefined \
             value, which would break the lower-level helper contract."
        );
    }

    /// Build a default `GenerateArgs` for testing the resolution helper.
    /// The `Default` derive isn't on the struct (clap derives don't include
    /// Default), so we construct one manually here.
    fn test_args_default() -> cli::GenerateArgs {
        cli::GenerateArgs {
            model: std::path::PathBuf::from("/dev/null"),
            prompt: None,
            prompt_file: None,
            tokenizer: None,
            config: None,
            temperature: 0.0,
            top_p: 1.0,
            top_k: 0,
            min_p: 0.0,
            repetition_penalty: 1.0,
            max_tokens: 1,
            mmproj: None,
            image: None,
            chat_template: None,
            chat_template_file: None,
            benchmark: false,
            speculative: false,
            kv_bits: None,
            enable_thinking: false,
            no_thinking: false,
            ignore_eos: false,
        }
    }

    // ---- ADR-005 iter-229 AC-parity: shared chat-template env ---------
    //
    // Both renderers build their environment via `build_chat_template_env`;
    // these pin the one-shot (Lenient) semantics the API path must not
    // disturb, plus the Strict policy the API path relies on.

    #[test]
    fn iter229_parity_tojson_non_finite_renders_null() {
        // Gate-3 #6 honesty note: serde_json serializes non-finite floats
        // as `null` itself, so this pins the RENDERED OUTCOME (`null`)
        // rather than proving the `unwrap_or_else("null")` fallback
        // fires — that branch stays as defense-in-depth for values whose
        // serialization genuinely errors.
        let mut env = build_chat_template_env(RaisePolicy::Lenient);
        env.add_template("t", "{{ x | tojson }}").unwrap();
        let out = env
            .get_template("t")
            .unwrap()
            .render(minijinja::context! { x => f64::NAN })
            .unwrap();
        assert_eq!(out, "null");
    }

    #[test]
    fn iter229_parity_pycompat_strip_resolves() {
        let mut env = build_chat_template_env(RaisePolicy::Lenient);
        env.add_template("t", "{{ ' x '.strip() }}").unwrap();
        let out = env.get_template("t").unwrap().render(()).unwrap();
        assert_eq!(out, "x");
    }

    #[test]
    fn iter229_parity_raise_exception_lenient_warns_and_continues() {
        let mut env = build_chat_template_env(RaisePolicy::Lenient);
        env.add_template("t", "{{ raise_exception('boom') }}x")
            .unwrap();
        let out = env.get_template("t").unwrap().render(()).unwrap();
        assert_eq!(out, "x", "Lenient policy must render through the raise");
    }

    #[test]
    fn iter229_parity_raise_exception_strict_hard_errors_with_message() {
        let mut env = build_chat_template_env(RaisePolicy::Strict);
        env.add_template("t", "{{ raise_exception('boom') }}x")
            .unwrap();
        let err = env.get_template("t").unwrap().render(()).unwrap_err();
        assert!(err.to_string().contains("boom"), "err={err}");
    }

    // ---- canonical render-and-diff thinking-mode detection -----------
    //
    // 2026-05-02 (peer-audit-driven rewrite of the auto-detect signal,
    // /tmp/cfa-thinking-detect/peer-detection-report.md).
    //
    // The old name-substring heuristic was rejected by the user:
    //   "Thinking is not determined by having thinking in the model
    //    name. Thinking is determined by the model class and what is
    //    available in the gguf file."
    //
    // The canonical signal — used by llama.cpp at
    // /opt/llama.cpp/common/chat-diff-analyzer.cpp:319-401 and
    // /opt/llama.cpp/common/chat.cpp:244-257 — is to render the
    // resolved chat template TWICE with `enable_thinking=true` then
    // `=false`, then recover whether the enabled render actually opens a
    // reasoning block. Plain byte inequality is too broad for Qwen3:
    // llama.cpp's own test_template.py expects Qwen-Qwen3-0.6B auto to end
    // at `<|im_start|>assistant\n`, even though `reasoning=off` appends an
    // empty `<think></think>` suppressor.

    use super::template_supports_enable_thinking;

    #[test]
    fn template_supports_enable_thinking_true_for_qwen3_chatml() {
        // The bundled qwen3-chatml.jinja vendor template has explicit
        // `enable_thinking is defined and is false` branching at lines
        // 147-153. Render-and-diff MUST detect this.
        assert!(
            template_supports_enable_thinking(QWEN3_CHATML),
            "qwen3-chatml.jinja MUST be detected as thinking-capable; \
             render-and-diff against vendor lines 147-153 should differ"
        );
    }

    #[test]
    fn template_supports_enable_thinking_false_for_thinking_agnostic_template() {
        // A minimal Jinja that doesn't reference `enable_thinking` at
        // all renders identically regardless of the flag → should NOT
        // be detected as thinking-capable.
        let template = "{% for m in messages %}{{ m.role }}: {{ m.content }}\n\
                        {% endfor %}{% if add_generation_prompt %}assistant: {% endif %}";
        assert!(
            !template_supports_enable_thinking(template),
            "thinking-agnostic template MUST NOT be flagged"
        );
    }

    #[test]
    fn template_supports_enable_thinking_false_for_plain_qwen3_suppressor_template() {
        // Mirrors llama.cpp tools/server/tests/unit/test_template.py:
        // Qwen-Qwen3-0.6B with reasoning=auto ends at the bare assistant
        // generation prompt, while reasoning=off appends an empty
        // `<think></think>` suppressor. This is a real byte difference, but
        // it is NOT evidence that auto should open thinking.
        let template = "{%- for m in messages -%}\
                        <|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n\
                        {%- endfor -%}\
                        {%- if add_generation_prompt -%}\
                        <|im_start|>assistant\n\
                        {%- if enable_thinking is defined and enable_thinking is false -%}\
                        <think>\n\n</think>\n\n\
                        {%- endif -%}\
                        {%- endif -%}";
        assert!(
            !template_supports_enable_thinking(template),
            "a disabled-only empty think suppressor must not auto-enable thinking"
        );
    }

    #[test]
    fn template_supports_enable_thinking_false_for_gemma4_ara_channel_suppressor() {
        // 2026-05-17 regression — Gemma 4 ara-abliterated APEX-Q5_K_M ships
        // a GGUF chat template whose `if not enable_thinking` branch appends
        // `<|channel>thought\n<channel|>` as a suppressor primer for
        // generation.  Pre-fix `rendered_prompt_opens_thinking` returned
        // `true` here (pure byte-diff), defaulting to enable_thinking=true,
        // which left the model bare at `<|turn>model\n` and produced
        // argmax-drift `서히-own-` repetition loops on enumerative prompts
        // (operator's chemistry-Format probe).  Pin: this section-marker
        // suppressor is NOT a `<think>`-style open, so the detector must
        // return FALSE.
        let template = "{%- for m in messages -%}\
                        <|turn>{{ m.role }}\n{{ m.content }}<turn|>\n\
                        {%- endfor -%}\
                        {%- if add_generation_prompt -%}\
                        <|turn>model\n\
                        {%- if not (enable_thinking | default(false)) -%}\
                        <|channel>thought\n<channel|>\
                        {%- endif -%}\
                        {%- endif -%}";
        assert!(
            !template_supports_enable_thinking(template),
            "gemma4-ara `<|channel>thought<channel|>` suppressor must not \
             auto-enable thinking — section markers are not `<think>` opens"
        );
    }

    #[test]
    fn template_supports_enable_thinking_false_for_empty_branch_template() {
        // Edge case: a template that DOES reference `enable_thinking`
        // but has empty branches (no byte-level difference) renders
        // identically and must NOT be flagged. Pins the property that
        // we're checking effective output, not just textual mention.
        let template = "{% if enable_thinking %}{% endif %}\
                        {% for m in messages %}{{ m.content }}{% endfor %}";
        assert!(
            !template_supports_enable_thinking(template),
            "empty-branch template MUST NOT be flagged (output bytes \
             are identical for true vs false)"
        );
    }

    #[test]
    fn template_supports_enable_thinking_false_on_malformed_template() {
        // Malformed Jinja must NOT crash the auto-detect — graceful
        // false fallback so the caller's actual render attempt is the
        // one that surfaces the parse error.
        let template = "{% unclosed";
        assert!(
            !template_supports_enable_thinking(template),
            "malformed template must produce false (graceful), not panic"
        );
    }

    #[test]
    fn resolve_enable_thinking_default_no_template_returns_some_false() {
        // No flag, no template (e.g. fallback path) → safe default.
        let args = test_args_default();
        assert_eq!(resolve_enable_thinking(&args, None), Some(false));
    }

    #[test]
    fn resolve_enable_thinking_default_thinking_template_auto_enables() {
        // The user's expected behavior on the bundled qwen3-chatml.jinja:
        // default cmdline → render-and-diff detects thinking branches →
        // enable_thinking=true → model emits BOTH reasoning + answer.
        // This is the "we want both" path the user requested.
        let args = test_args_default();
        assert_eq!(
            resolve_enable_thinking(&args, Some(QWEN3_CHATML)),
            Some(true),
            "qwen3-chatml.jinja default render MUST auto-enable thinking"
        );
    }

    #[test]
    fn resolve_enable_thinking_default_non_thinking_template_returns_some_false() {
        // Plain ChatML without thinking branches → render-and-diff is
        // bytewise identical → no auto-enable → safe default false.
        let template = "{% for m in messages %}<|im_start|>{{ m.role }}\n\
                        {{ m.content }}<|im_end|>\n{% endfor %}\
                        {% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}";
        let args = test_args_default();
        assert_eq!(resolve_enable_thinking(&args, Some(template)), Some(false));
    }

    #[test]
    fn resolve_enable_thinking_explicit_enable_overrides_auto_detect() {
        // Explicit --enable-thinking wins even when the template would
        // auto-detect to false.
        let mut args = test_args_default();
        args.enable_thinking = true;
        let plain = "{{ messages[0].content }}";
        assert_eq!(resolve_enable_thinking(&args, Some(plain)), Some(true));
    }

    #[test]
    fn resolve_enable_thinking_explicit_no_thinking_overrides_auto_detect() {
        // Explicit --no-thinking wins even when the template would
        // auto-detect to true. Critical escape hatch: a template that
        // probes as thinking-capable but breaks for the user's specific
        // checkpoint must be opt-out-able.
        let mut args = test_args_default();
        args.no_thinking = true;
        assert_eq!(
            resolve_enable_thinking(&args, Some(QWEN3_CHATML)),
            Some(false)
        );
    }

    #[test]
    fn user_candy_regression_default_non_thinking_template_renders_pre_closed() {
        // 2026-05-02 candy-prompt regression: a non-thinking template
        // (no `enable_thinking` Jinja branch) auto-detects to false →
        // render goes through but `enable_thinking=false` is irrelevant
        // since the template doesn't branch on it. Pin: with QWEN3_CHATML
        // the regression-safe path requires explicit --no-thinking.
        let mut args = test_args_default();
        args.no_thinking = true;
        let resolved = resolve_enable_thinking(&args, Some(QWEN3_CHATML));
        let rendered = render_jinja_template(QWEN3_CHATML, "make me candy", resolved)
            .expect("--no-thinking render must succeed");
        assert!(
            rendered.ends_with("<|im_start|>assistant\n<think>\n\n</think>\n\n"),
            "--no-thinking on qwen3-chatml MUST emit pre-closed think block. \
             Actual tail: `{}`",
            &rendered[rendered.len().saturating_sub(80)..]
        );
    }

    #[test]
    fn user_we_want_both_default_on_qwen3_chatml_renders_open_block() {
        // The "we want both" path: with the bundled qwen3-chatml.jinja
        // (auto-detected as thinking-capable via render-and-diff), the
        // default cmdline MUST emit `<think>\n` so the model produces
        // BOTH reasoning AND answer — exactly what the user asked for.
        // FALSIFIES if a future change reverts to name-substring or
        // unconditional false.
        let args = test_args_default();
        let resolved = resolve_enable_thinking(&args, Some(QWEN3_CHATML));
        assert_eq!(resolved, Some(true), "auto-detect must enable thinking");
        let rendered = render_jinja_template(QWEN3_CHATML, "make me candy", resolved)
            .expect("default render must succeed");
        assert!(
            rendered.ends_with("<|im_start|>assistant\n<think>\n"),
            "default cmdline on qwen3-chatml MUST emit open think block. \
             Actual tail: `{}`",
            &rendered[rendered.len().saturating_sub(80)..]
        );
    }

    #[test]
    fn render_jinja_qwen3_chatml_enable_thinking_does_not_affect_user_message() {
        // Sanity: the user message body itself must be present regardless of
        // the thinking flag. Pin against accidental message stripping.
        for et in [None, Some(true), Some(false)] {
            let rendered = render_jinja_template(QWEN3_CHATML, "MARKER_USER_BODY_42", et)
                .expect("render must succeed");
            assert!(
                rendered.contains("MARKER_USER_BODY_42"),
                "user message body missing from rendered prompt with \
                 enable_thinking={et:?}"
            );
            assert!(
                rendered.contains("<|im_start|>user\n"),
                "user-turn opener missing from rendered prompt with \
                 enable_thinking={et:?}"
            );
        }
    }

    // ---- run_decode_loop matrix tests --------------------------------
    //
    // 2026-05-02 — H3 Layer A unlock per item (c) of the dossier plan
    // (docs/research/decode-test-gap-2026-05-02.md). The decode-loop
    // body was previously inline in cmd_generate_qwen35; today's 12+
    // helper tests pinned individual pure functions in isolation but
    // not the loop body that composes them. These tests feed synthetic
    // token streams through `run_decode_loop` and assert on the
    // `DecodeLoopOutcome` + emitted-event sequence — the entire loop
    // is now testable without a Metal GPU.

    /// Build a closure factory that returns canned tokens in order.
    /// `tokens[0]` is returned on the first step_model call, `tokens[1]`
    /// on the second, etc. Panics if the loop calls past the supplied
    /// stream — that's a test-author bug.
    fn canned_step_model(tokens: Vec<u32>) -> impl FnMut(u32, i32, &[u32]) -> super::Result<u32> {
        let mut iter = tokens.into_iter();
        move |_prev, _pos, _generated| {
            let t = iter.next().expect("test bug: canned stream exhausted");
            Ok(t)
        }
    }

    /// Decode tokens to text by trivial mapping: each id maps to the
    /// digit/letter that visually identifies it. Avoids needing a real
    /// tokenizer in unit tests.
    fn ascii_decode(toks: &[u32]) -> String {
        toks.iter().map(|&t| char::from(t as u8)).collect()
    }

    #[test]
    fn run_decode_loop_max_tokens_reached() {
        // No EOS, no leak, no repetition — loop runs to budget.
        let outcome = run_decode_loop(
            b'A' as u32,
            10,
            1024,
            5,
            &[999],
            canned_step_model(vec![b'B' as u32, b'C' as u32, b'D' as u32, b'E' as u32]),
            ascii_decode,
            |_event| {},
        )
        .expect("step_model never errors here");
        assert_eq!(outcome.stop_reason, DecodeStopReason::MaxTokensReached);
        // 1 (seed) + 4 (step 1..5) = 5 generated.
        assert_eq!(outcome.generated, 5);
    }

    #[test]
    fn run_decode_loop_stops_on_integer_eos() {
        // Stream produces a token id that's in eos_token_ids.
        let outcome = run_decode_loop(
            b'A' as u32,
            10,
            1024,
            10,
            &[42],
            canned_step_model(vec![b'B' as u32, 42, b'D' as u32]),
            ascii_decode,
            |_event| {},
        )
        .expect("ok");
        assert_eq!(outcome.stop_reason, DecodeStopReason::EosTokenId(42));
        // Seed (A) + step 1 (B) + step 2 (42, EOS-detected at TOP of step 3) = 3.
        assert_eq!(outcome.generated, 3);
    }

    #[test]
    fn run_decode_loop_stops_on_special_token_leak() {
        // Stream produces a sequence whose decoded text contains
        // `<|im_start|>user` (one of SPECIAL_TOKEN_STOPS — the model
        // hallucinating the next user turn = its assistant turn is over).
        // Use a custom decode_text that injects the leak after step 2.
        let mut step = 0;
        let outcome = run_decode_loop(
            b'A' as u32,
            10,
            1024,
            10,
            &[999],
            canned_step_model(vec![b'B' as u32, b'C' as u32, b'D' as u32]),
            move |toks| {
                step += 1;
                let mut s: String = toks.iter().map(|&t| char::from(t as u8)).collect();
                if step >= 3 {
                    s.push_str("<|im_start|>user");
                }
                s
            },
            |_event| {},
        )
        .expect("ok");
        assert_eq!(
            outcome.stop_reason,
            DecodeStopReason::SpecialTokenLeak("<|im_start|>user"),
        );
    }

    #[test]
    fn run_decode_loop_streams_raw_deltas_then_stops_on_full_marker() {
        // 2026-05-03 — the inference layer now emits RAW decoded text. As
        // `<|im_start|>user` arrives byte-by-byte, each partial prefix is
        // streamed to the caller AS-IS. Once the full marker assembles in
        // the cumulative text, the SpecialTokenLeak stop fires.
        //
        // Pre-2026-05-03 this test asserted that partial stop prefixes
        // were buffered (held back via `trailing_special_token_prefix_len`)
        // until the marker resolved. That visible-prefix surgery was
        // removed because it leaked into a 150-LOC web of pattern matches
        // that the user ("VERY brittle patterns / with the stripping")
        // correctly identified as inference/display layer confusion.
        // llama.cpp's libllama doesn't do that surgery either; the CLI /
        // application layer is responsible for any user-facing formatting.
        let mut deltas: Vec<String> = Vec::new();
        const TARGET: &str = "<|im_start|>user";
        let outcome = run_decode_loop(
            b'<' as u32,
            10,
            1024,
            32,
            &[999],
            canned_step_model(TARGET.bytes().skip(1).map(|b| b as u32).collect()),
            ascii_decode,
            |event| deltas.push(event.delta.to_string()),
        )
        .expect("ok");
        assert_eq!(
            outcome.stop_reason,
            DecodeStopReason::SpecialTokenLeak("<|im_start|>user")
        );
        // Deltas concatenate to the FULL marker text — partial prefixes
        // ARE streamed (raw inference layer; caller decides whether to
        // display them).
        let joined: String = deltas.concat();
        assert_eq!(joined, TARGET);
    }

    #[test]
    fn run_decode_loop_stops_on_repetition() {
        // Build a stream that's identical 8-token cycles repeated.
        let cycle: Vec<u32> = (b'A'..=b'H').map(|c| c as u32).collect();
        let mut stream: Vec<u32> = Vec::new();
        for _ in 0..40 {
            stream.extend(cycle.iter().copied());
        }
        let outcome = run_decode_loop(
            b'Z' as u32,
            10,
            1024,
            500,
            &[999],
            canned_step_model(stream),
            ascii_decode,
            |_event| {},
        )
        .expect("ok");
        match outcome.stop_reason {
            DecodeStopReason::RepetitionLoop { ngram, repeats } => {
                assert!(ngram >= 8, "expected ngram >= 8, got {ngram}");
                // In the streaming decode loop the detector fires exactly at
                // MIN_OCCURRENCES_CONTENT (3) — the loop stops before more
                // cycles can accumulate. `repeats` reports the actual observed
                // count at firing time, which for streaming is always the
                // threshold. (Pre-9f797761 the threshold was 4; the relaxation
                // to 3 lowered this assertion accordingly.)
                assert!(repeats >= 3, "expected repeats >= 3, got {repeats}");
            }
            other => panic!("expected RepetitionLoop, got {other:?}"),
        }
    }

    #[test]
    fn run_decode_loop_stops_on_max_seq_overrun() {
        // prompt_len + step - 1 == max_seq triggers MaxSeq stop. With
        // prompt_len=8, max_seq=10, step=1 puts pos=8, step=2 puts
        // pos=9, step=3 puts pos=10 → overrun.
        let outcome = run_decode_loop(
            b'A' as u32,
            8,
            10,
            10,
            &[999],
            canned_step_model(vec![b'B' as u32, b'C' as u32, b'D' as u32]),
            ascii_decode,
            |_event| {},
        )
        .expect("ok");
        assert_eq!(outcome.stop_reason, DecodeStopReason::MaxSeq);
        // Seed (A) + step 1 (B) + step 2 (C) = 3; step 3 hits the
        // pos-check at top and returns MaxSeq before another forward call.
        assert_eq!(outcome.generated, 3);
    }

    #[test]
    fn run_decode_loop_emits_step_zero_seed_event() {
        // Pin: the seed token from prefill argmax MUST be emitted as
        // step=0 with delta == cumulative_text. This is what makes the
        // FIRST printed bytes appear in the production CLI without
        // requiring a special-case before the loop.
        let mut events: Vec<(usize, u32, String, String)> = Vec::new();
        let _ = run_decode_loop(
            b'A' as u32,
            10,
            1024,
            2,
            &[999],
            canned_step_model(vec![b'B' as u32]),
            ascii_decode,
            |event| {
                events.push((
                    event.step,
                    event.token,
                    event.cumulative_text.to_string(),
                    event.delta.to_string(),
                ));
            },
        )
        .expect("ok");
        assert_eq!(events.len(), 2, "expected 2 events (step 0 + step 1)");
        assert_eq!(events[0].0, 0);
        assert_eq!(events[0].1, b'A' as u32);
        assert_eq!(events[0].2, "A");
        assert_eq!(events[0].3, "A", "step 0 delta MUST equal cumulative");
    }

    #[test]
    fn run_decode_loop_delta_is_cumulative_minus_previous() {
        // Pin: per-step delta == cumulative_text - previous cumulative.
        // For UTF-8 streaming correctness this is THE invariant.
        let mut deltas: Vec<String> = Vec::new();
        let _ = run_decode_loop(
            b'X' as u32,
            10,
            1024,
            4,
            &[999],
            canned_step_model(vec![b'Y' as u32, b'Z' as u32, b'!' as u32]),
            ascii_decode,
            |event| deltas.push(event.delta.to_string()),
        )
        .expect("ok");
        // step 0 emits "X" (full seed); step 1 emits "Y"; step 2 "Z"; step 3 "!".
        assert_eq!(deltas, vec!["X", "Y", "Z", "!"]);
    }

    #[test]
    fn run_decode_loop_propagates_step_model_error() {
        // A failing forward pass MUST surface the error through `Result`.
        let outcome = run_decode_loop(
            b'A' as u32,
            10,
            1024,
            5,
            &[999],
            |_prev, pos, _generated| -> super::Result<u32> {
                Err(anyhow::anyhow!("synthetic forward failure at pos {pos}"))
            },
            ascii_decode,
            |_event| {},
        );
        assert!(outcome.is_err());
        let msg = format!("{:#}", outcome.unwrap_err());
        assert!(
            msg.contains("synthetic forward failure"),
            "expected error to propagate, got: {msg}"
        );
    }

    #[test]
    fn run_decode_loop_zero_max_tokens_emits_seed_only() {
        // max_tokens=1 means the for-loop range is 1..1 (empty); only the
        // seed event fires, and we exit with MaxTokensReached + 1 generated.
        let mut events_count = 0usize;
        let outcome = run_decode_loop(
            b'A' as u32,
            10,
            1024,
            1,
            &[999],
            canned_step_model(vec![]), // empty — never called
            ascii_decode,
            |_event| events_count += 1,
        )
        .expect("ok");
        assert_eq!(events_count, 1, "only the seed event must fire");
        assert_eq!(outcome.generated, 1);
        assert_eq!(outcome.stop_reason, DecodeStopReason::MaxTokensReached);
    }

    #[test]
    fn run_decode_loop_initial_token_is_eos_stops_at_top_of_step_one() {
        // Edge case: prefill argmax returns an EOS-class token. The loop
        // emits the seed event at step 0, then enters step 1, sees
        // next_token in eos_token_ids, and stops. This matches production
        // semantics (`for step in 1..` with the eos check at the top).
        let mut events_count = 0usize;
        let outcome = run_decode_loop(
            42,
            10,
            1024,
            5,
            &[42],
            canned_step_model(vec![]),
            ascii_decode,
            |_event| events_count += 1,
        )
        .expect("ok");
        assert_eq!(events_count, 1);
        assert_eq!(outcome.generated, 1);
        assert_eq!(outcome.stop_reason, DecodeStopReason::EosTokenId(42));
    }

    #[test]
    fn run_decode_loop_phi3_end_marker_caught_via_special_token_leak() {
        // 2026-05-02 user-reported regression class: the model emits
        // `<|end|>` as a multi-token text fragment after thinking content.
        // run_decode_loop must catch this via SpecialTokenLeak ("<|end|>").
        let mut step = 0;
        let outcome = run_decode_loop(
            b'A' as u32,
            10,
            1024,
            10,
            &[999],
            canned_step_model(vec![b'B' as u32, b'C' as u32, b'D' as u32]),
            move |toks| {
                step += 1;
                let mut s: String = toks.iter().map(|&t| char::from(t as u8)).collect();
                if step >= 4 {
                    s.push_str("<|end|>");
                }
                s
            },
            |_event| {},
        )
        .expect("ok");
        assert_eq!(
            outcome.stop_reason,
            DecodeStopReason::SpecialTokenLeak("<|end|>")
        );
    }

    // ─────────────────────────────────────────────────────────────────────
    // ADR-040 Phase C iter-4 (C4) — CLI/env wiring for `HF2Q_SCHEDULER` +
    // `--scheduler` / `HF2Q_MAX_SLOTS` + `--max-slots` (2026-05-23).
    //
    // These tests exercise the pure-function `parse_scheduler_config`
    // helper that `cmd_serve` calls — no env mutation, no listener bind,
    // no model load. The helper is the operator-facing contract surface:
    // any future drift in defaults / precedence / validation falsifies
    // here before the binary ships.
    //
    // Contract pinned (per ADR-040 §3.4 + §3.6):
    //   - Env-absence (CLI + env both unset) → `EngineMode::SerialFifo`
    //     (the ADR-005 byte-equivalent default).
    //   - `fifo_serial` / `inflight_batched` parse case-insensitively
    //     from the env-var layer.
    //   - `max_slots` defaults to `DEFAULT_MAX_SLOTS_UNDER_INFLIGHT = 4`
    //     under `inflight_batched`; ignored entirely under `fifo_serial`
    //     (legacy single-slot worker).
    //   - `max_slots == 0` (CLI or env) is REJECTED loudly with an
    //     operator-actionable message (per ADR-040 iter-2.5 F3a
    //     `.max(1)` discipline applied at the CLI layer instead of
    //     silently coerced).
    //   - Unknown scheduler value via env rejected with named-supported
    //     diagnostic.
    // ─────────────────────────────────────────────────────────────────────

    /// ADR-040 §3.6 — backward-compat: with CLI + env both unset, the
    /// resolved mode is `EngineMode::SerialFifo`. This is the
    /// load-bearing pre-ADR-040 byte-equivalence pin at the CLI layer.
    #[test]
    fn c4_scheduler_env_unset_defaults_to_fifo_serial() {
        let mode = parse_scheduler_config(None, None, None, None)
            .expect("env-absence must yield SerialFifo, not an error");
        assert_eq!(
            mode,
            crate::serve::api::engine::EngineMode::SerialFifo,
            "ADR-040 §3.6: env-absence MUST be byte-equivalent to \
             pre-ADR-040 (= EngineMode::SerialFifo). Got {mode:?}."
        );
    }

    /// ADR-040 §3.6 — explicit `HF2Q_SCHEDULER=fifo_serial` matches the
    /// default (lowercase canonical form).
    #[test]
    fn c4_scheduler_env_fifo_serial_lowercase_matches() {
        let mode = parse_scheduler_config(None, Some("fifo_serial"), None, None)
            .expect("HF2Q_SCHEDULER=fifo_serial must parse");
        assert_eq!(mode, crate::serve::api::engine::EngineMode::SerialFifo);
    }

    /// `HF2Q_SCHEDULER=inflight_batched` resolves to
    /// `EngineMode::SlotAware { max_slots: DEFAULT_MAX_SLOTS_UNDER_INFLIGHT }`
    /// (the env-var layer accepts the same names clap exposes via
    /// `--scheduler`, case-insensitive).
    #[test]
    fn c4_scheduler_env_inflight_batched_matches() {
        let mode = parse_scheduler_config(None, Some("inflight_batched"), None, None)
            .expect("HF2Q_SCHEDULER=inflight_batched must parse");
        assert_eq!(
            mode,
            crate::serve::api::engine::EngineMode::SlotAware {
                max_slots: DEFAULT_MAX_SLOTS_UNDER_INFLIGHT,
            }
        );
    }

    /// The env-var path is case-insensitive — operators occasionally
    /// export env vars in upper-case by convention. Mirrors the
    /// clap-`ValueEnum`-default case-insensitivity on the CLI side.
    #[test]
    fn c4_scheduler_env_case_insensitive() {
        // Upper-case
        let mode = parse_scheduler_config(None, Some("INFLIGHT_BATCHED"), None, None)
            .expect("HF2Q_SCHEDULER=INFLIGHT_BATCHED must parse case-insensitively");
        assert!(matches!(
            mode,
            crate::serve::api::engine::EngineMode::SlotAware { .. }
        ));
        // Mixed case
        let mode = parse_scheduler_config(None, Some("FiFo_SeRiAl"), None, None)
            .expect("HF2Q_SCHEDULER=FiFo_SeRiAl must parse case-insensitively");
        assert_eq!(mode, crate::serve::api::engine::EngineMode::SerialFifo);
        // Whitespace-only treated as unset → default SerialFifo
        let mode = parse_scheduler_config(None, Some("   "), None, None)
            .expect("HF2Q_SCHEDULER=`   ` must be treated as unset");
        assert_eq!(mode, crate::serve::api::engine::EngineMode::SerialFifo);
    }

    /// Unknown values via the env-var path are REJECTED with a clear
    /// operator-actionable message that names the supported set. The
    /// CLI path is gated by clap's `ValueEnum` so this only fires from
    /// env. Matches the "fail loudly, no silent fallback" stance ADR-040
    /// §7 mantra demands.
    #[test]
    fn c4_scheduler_env_unknown_value_errors() {
        let err = parse_scheduler_config(None, Some("foo"), None, None)
            .expect_err("unknown HF2Q_SCHEDULER value must error");
        assert!(err.contains("HF2Q_SCHEDULER"), "msg: {err}");
        assert!(err.contains("foo"), "msg must echo bad value: {err}");
        assert!(
            err.contains("fifo_serial") && err.contains("inflight_batched"),
            "msg must name supported values: {err}"
        );
    }

    /// ADR-040 §3.4 — `max_slots` default = `4` under
    /// `inflight_batched`. Operator deploys ramp 1 with headroom (half
    /// the ≥8 reopen-trigger).
    #[test]
    fn c4_max_slots_env_unset_defaults_to_4_under_inflight() {
        let mode = parse_scheduler_config(None, Some("inflight_batched"), None, None)
            .expect("inflight_batched without HF2Q_MAX_SLOTS must default");
        assert_eq!(
            mode,
            crate::serve::api::engine::EngineMode::SlotAware { max_slots: 4 },
            "ADR-040 §3.4: default max_slots under inflight_batched MUST be 4"
        );
        assert_eq!(
            DEFAULT_MAX_SLOTS_UNDER_INFLIGHT, 4,
            "ADR-040 §3.4: the named constant MUST hold the 4 default"
        );
    }

    /// `max_slots` is IGNORED on the `fifo_serial` path (the legacy
    /// 3-arg `Engine::spawn` is single-slot by definition; `max_slots`
    /// is a SlotAware-only concept). Even if the operator supplies
    /// `--max-slots 8 --scheduler fifo_serial`, the resolved mode is
    /// `EngineMode::SerialFifo` with no `max_slots` field.
    #[test]
    fn c4_max_slots_env_unset_defaults_to_1_under_fifo_serial() {
        // CLI says `--max-slots 8` but env+cli say SerialFifo → SerialFifo
        // wins, max_slots is IGNORED.
        let mode = parse_scheduler_config(Some(cli::SchedulerArg::FifoSerial), None, Some(8), None)
            .expect("max_slots on fifo_serial must be ignored, not error");
        assert_eq!(
            mode,
            crate::serve::api::engine::EngineMode::SerialFifo,
            "ADR-040: under fifo_serial, max_slots is ignored — \
             resolved mode MUST be SerialFifo (single-slot by definition). \
             Got {mode:?}."
        );

        // Same conclusion when BOTH are unset (the default-default path).
        let mode2 = parse_scheduler_config(None, None, None, None).expect("default ok");
        assert_eq!(mode2, crate::serve::api::engine::EngineMode::SerialFifo);
    }

    /// ADR-040 iter-2.5 F3a discipline — `max_slots == 0` is REJECTED
    /// at the CLI layer rather than silently coerced via `.max(1)`. The
    /// fail-loud stance is the ADR-040 §7 mantra requirement; the
    /// `.max(1)` defense-in-depth still lives in
    /// `parse_scheduler_config`'s SlotAware arm in case a future
    /// refactor loses this check.
    #[test]
    fn c4_max_slots_env_zero_normalizes_or_errors() {
        // CLI-supplied --max-slots 0 → error
        let err = parse_scheduler_config(
            Some(cli::SchedulerArg::InflightBatched),
            None,
            Some(0),
            None,
        )
        .expect_err("max_slots=0 must be rejected, not silently coerced");
        assert!(err.contains("max-slots"), "msg: {err}");
        assert!(err.contains('0'), "msg must echo the bad value: {err}");
        assert!(
            err.contains("F3a") || err.contains(".max(1)"),
            "msg should cite the iter-2.5 F3a discipline: {err}"
        );

        // Env-supplied HF2Q_MAX_SLOTS=0 → error
        let err = parse_scheduler_config(None, Some("inflight_batched"), None, Some("0"))
            .expect_err("HF2Q_MAX_SLOTS=0 must be rejected, not silently coerced");
        assert!(err.contains("max-slots"), "msg: {err}");

        // Bonus: non-parseable env value also errors with named diag.
        let err = parse_scheduler_config(None, None, None, Some("not-a-number"))
            .expect_err("non-u32 HF2Q_MAX_SLOTS must error");
        assert!(err.contains("HF2Q_MAX_SLOTS"), "msg: {err}");
        assert!(err.contains("u32"), "msg must name u32: {err}");
    }

    // ─────────────────────────────────────────────────────────────────────
    // Precedence + interaction pins — defense-in-depth around the
    // CLI-wins-over-env contract. Not in the brief's required set, but
    // they fall out of the same helper and pin behavior the operator
    // would discover via surprise otherwise.
    // ─────────────────────────────────────────────────────────────────────

    /// CLI flag wins over env (mirrors `--auth-token` / `HF2Q_AUTH_TOKEN`
    /// precedence cmd_serve already uses).
    #[test]
    fn c4_scheduler_cli_wins_over_env() {
        // CLI says fifo_serial, env says inflight_batched → CLI wins.
        let mode = parse_scheduler_config(
            Some(cli::SchedulerArg::FifoSerial),
            Some("inflight_batched"),
            None,
            None,
        )
        .expect("cli wins, must resolve to SerialFifo");
        assert_eq!(mode, crate::serve::api::engine::EngineMode::SerialFifo);

        // Reverse direction — CLI says inflight_batched, env says
        // fifo_serial → CLI wins.
        let mode = parse_scheduler_config(
            Some(cli::SchedulerArg::InflightBatched),
            Some("fifo_serial"),
            Some(7),
            None,
        )
        .expect("cli wins, must resolve to SlotAware{7}");
        assert_eq!(
            mode,
            crate::serve::api::engine::EngineMode::SlotAware { max_slots: 7 }
        );
    }

    /// CLI `--max-slots` wins over `HF2Q_MAX_SLOTS`.
    #[test]
    fn c4_max_slots_cli_wins_over_env() {
        let mode = parse_scheduler_config(
            Some(cli::SchedulerArg::InflightBatched),
            None,
            Some(2),
            Some("16"),
        )
        .expect("cli max_slots wins over env");
        assert_eq!(
            mode,
            crate::serve::api::engine::EngineMode::SlotAware { max_slots: 2 }
        );
    }
}