franken_ocr 0.10.0

//! Pure-Rust byte-level BPE tokenizer over `tokenizer.json` ([SPEC-019,
//! SPEC-035], OQ-16). Token-id-exact vs HF `LlamaTokenizerFast` — the L0/L4
//! prerequisite for every downstream conformance gate (AGENTS.md testing
//! policy): a single mismatched id corrupts the whole parity ladder.
//!
//! The algorithm is GPT-2-style **byte-level BPE** (NOT SentencePiece, NOT
//! Llama): a no-op normalizer, a four-stage byte-level pre-tokenizer
//! ([`pretok`]), merge-rank BPE over the 256-symbol byte alphabet, and an
//! added/special-token table that is split out of the text *before* BPE (HF
//! `AddedVocabulary` semantics). There is no UNK and no algorithmic
//! byte-fallback: full byte coverage comes from the byte→unicode remap
//! (OQ-16 §2).
//!
//! BOS/EOS policy (OQ-16 §5): we do **not** auto-append EOS and do **not**
//! auto-prepend BOS inside [`Tokenizer::encode`] (it encodes with
//! `add_special_tokens=False`, matching `modeling_unlimitedocr.py:260`). The
//! prompt builder prepends a single id-0 BOS at the very front of the final
//! sequence; the special-token ids it needs are exposed via [`special`] and the
//! [`Tokenizer`] accessors.

use std::collections::HashMap;
use std::path::Path;

use serde::Deserialize;

use crate::error::{FocrError, FocrResult};

pub mod music;
mod ops;
/// Pure pretokenizers (public for the fuzz harness — bd-10sb.1: the
/// hand-rolled Unicode split classes are an untrusted-input surface; the
/// fuzz targets assert totality + lossless re-join over arbitrary text).
pub mod pretok;
pub mod tiktoken;
mod unicode_tables;

pub use ops::TokenizerOps;

/// SmolVLM2 / SmolLM2 special-token ids (C6, bd-3jo6.3.6) — pinned by
/// `docs/zoo/smolvlm2-spec.md` §5 (verified against the released
/// `tokenizer.json` + `added_tokens.json` + `generation_config.json`) and
/// cross-checked against the loaded added-token table under the
/// [`PretokScheme::SmolLm2`] scheme.
pub mod special_smollm2 {
    /// `<|endoftext|>` — the unk token (id 0). `generation_config.json` also
    /// labels it bos, but the chat template supplies `<|im_start|>` literally
    /// and nothing auto-prepends (spec §5/§7 — the disagreement is harmless).
    pub const UNK: u32 = 0;
    /// `<|im_start|>` — the tokenizer's bos (`tokenizer_config.json`).
    pub const BOS: u32 = 1;
    /// `<|im_end|>` — the tokenizer's pad (`text_config.pad_token_id` = 2).
    pub const PAD: u32 = 2;
    /// `<global-img>` — brackets the global (thumbnail-LAST) frame.
    pub const GLOBAL_IMG: u32 = 49152;
    /// `<fake_token_around_image>` — the image-expansion bracket token.
    pub const FAKE_AROUND_IMAGE: u32 = 49189;
    /// `<image>` — the splice slot id (`image_token_id`); 64 per 512² frame.
    pub const IMAGE: u32 = 49190;
    /// `<end_of_utterance>` — the generation stop id (eos, 49279).
    pub const END_OF_UTTERANCE: u32 = 49279;
}

/// OneChart / OPT special-token ids (D9, bd-3jo6.4.9) — pinned by
/// `docs/zoo/onechart-spec.md` §4/§7 (verified against `added_tokens.json` +
/// `config.json`) and cross-checked against the loaded table under the
/// [`PretokScheme::Gpt2`] scheme.
pub mod special_opt {
    /// `<pad>` (OPT config `pad_token_id`).
    pub const PAD: u32 = 1;
    /// `</s>` — OPT's bos AND eos (config `bos/eos_token_id` = 2). The prompt
    /// builder owns the leading `</s>`; encode adds nothing.
    pub const BOS_EOS: u32 = 2;
    /// `<imgpad>` — the 256 vision splice slots (GOT lineage).
    pub const IMG_PAD: u32 = 50_265;
    /// `<img>` opening bracket.
    pub const IMG_START: u32 = 50_266;
    /// `</img>` closing bracket.
    pub const IMG_END: u32 = 50_267;
    /// `<Number>` — the number-head trigger token (census §8).
    pub const NUMBER: u32 = 50_268;
}

/// Hardcoded special-token ids ([SPEC-014/019]). These are pinned by the model
/// runtime (`modeling_unlimitedocr.py`) and cross-checked against
/// `tokenizer.json .added_tokens`; the loader asserts the loaded table agrees.
pub mod special {
    /// `<｜begin▁of▁sentence｜>` ([SPEC-014]). Note the fullwidth bar U+FF5C and
    /// the bullet U+2581 — DeepSeek-V2 glyphs, NOT ASCII `|`/`_`.
    pub const BOS: u32 = 0;
    /// `<｜end▁of▁sentence｜>` ([SPEC-014]).
    pub const EOS: u32 = 1;
    /// `<｜▁pad▁｜>` ([SPEC-014]).
    pub const PAD: u32 = 2;
    /// `<image>` ([SPEC-019]); the runtime hardcodes this id
    /// (`modeling_unlimitedocr.py:845 image_token_id = 128815`).
    pub const IMAGE: u32 = 128815;
    /// `<|ref|>` ([SPEC-019]).
    pub const REF: u32 = 128816;
    /// `<|/ref|>` ([SPEC-019]).
    pub const REF_END: u32 = 128817;
    /// `<|det|>` ([SPEC-019]).
    pub const DET: u32 = 128818;
    /// `<|/det|>` ([SPEC-019]).
    pub const DET_END: u32 = 128819;
    /// `<|grounding|>` ([SPEC-019]).
    pub const GROUNDING: u32 = 128820;
    /// `<td>` ([SPEC-019]).
    pub const TD: u32 = 128821;
    /// `</td>` ([SPEC-019]).
    pub const TD_END: u32 = 128822;
    /// `<tr>` ([SPEC-019]).
    pub const TR: u32 = 128823;
    /// `</tr>` ([SPEC-019]).
    pub const TR_END: u32 = 128824;
    /// `<|User|>` ([SPEC-019]); ASCII-bar variant (distinct from `<｜User｜>`).
    pub const USER: u32 = 128825;
    /// `<|Assistant|>` ([SPEC-019]); ASCII-bar variant.
    pub const ASSISTANT: u32 = 128826;
    /// `<PAGE>` — Unlimited-OCR-2 only (absent from the v1 `tokenizer.json`):
    /// prefixes each page's `<image>` in the prompt and separates pages in the
    /// output. Non-special, so `skip_special` decoding keeps it (U2-A4).
    pub const PAGE: u32 = 128827;
    /// `<BLOCK>` — Unlimited-OCR-2 only, non-special. Inference never reads it;
    /// the model emits it to open each output block, which the postprocess
    /// parses as text (U2-E3b).
    pub const BLOCK: u32 = 128828;
    /// `<FORK>` — Unlimited-OCR-2 only, non-special. The model emits it between
    /// a block's prefix and its text (U2-E3b).
    pub const FORK: u32 = 128829;
    /// `<CHILD>` — Unlimited-OCR-2 only, non-special; in none of the oracle
    /// outputs. The highest id the v2 tokenizer produces (v1's is [`ASSISTANT`]).
    pub const CHILD: u32 = 128830;
}

// ── `tokenizer.json` deserialization (only the fields we need) ──────────────

/// Top-level `tokenizer.json` shape (subset). Unused sections (`normalizer`,
/// `decoder`, `post_processor`, `padding`, `truncation`, `version`) are ignored
/// — the encode path replicates them in code (no-op normalizer, byte-level
/// decoder, BOS-only post-processor that we do NOT apply, OQ-16 §3-5). The
/// `pre_tokenizer` section IS read: it selects the [`PretokScheme`].
#[derive(Debug, Deserialize)]
struct RawTokenizer {
    #[serde(default)]
    added_tokens: Vec<RawAddedToken>,
    model: RawModel,
    #[serde(default)]
    pre_tokenizer: Option<serde_json::Value>,
}

/// One entry of `.added_tokens` — an added/special token spliced out of the
/// text before BPE.
#[derive(Debug, Deserialize)]
struct RawAddedToken {
    id: u32,
    content: String,
    /// `special:true` for the OCR/role glyphs; `false` for the DeepSeek
    /// `<｜User｜>`-style glyphs. We split on BOTH (HF `AddedVocabulary` splits on
    /// every added token regardless of the `special` flag); the flag is retained
    /// only for the decoder's `skip_special_tokens` behavior.
    #[serde(default)]
    special: bool,
}

/// The `.model` section — a `BPE` with `vocab` + `merges`.
#[derive(Debug, Deserialize)]
struct RawModel {
    vocab: HashMap<String, u32>,
    /// `merges` is either the new array-of-pairs format `[["Ġ","t"], …]` or the
    /// legacy space-joined-string format `["Ġ t", …]`. We accept both.
    #[serde(default)]
    merges: Vec<RawMerge>,
}

/// A single merge rule, tolerant of both `tokenizer.json` merge encodings.
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum RawMerge {
    /// New format: `["left", "right"]`.
    Pair([String; 2]),
    /// Legacy format: `"left right"` (single space-separated string).
    Joined(String),
}

impl RawMerge {
    /// Normalize to a `(left, right)` pair. The legacy joined form splits on the
    /// FIRST space only (a byte-level token can itself be `"Ġ"`, never contains
    /// an interior space, so first-space split is unambiguous).
    fn into_pair(self) -> Option<(String, String)> {
        match self {
            RawMerge::Pair([l, r]) => Some((l, r)),
            RawMerge::Joined(s) => {
                let mut it = s.splitn(2, ' ');
                let l = it.next()?.to_string();
                let r = it.next()?.to_string();
                Some((l, r))
            }
        }
    }
}

// ── Pre-tokenizer scheme selection ──────────────────────────────────────────

/// Which pre-tokenizer sequence the loaded `tokenizer.json` declares. The BPE
/// core, added-token splitting, and byte-level decode are shared; only the
/// pre-BPE splitting (and the pinned special-id table) differ per scheme.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PretokScheme {
    /// The DeepSeek/Llama 4-stage `Split` sequence + `ByteLevel(use_regex=false)`
    /// — Baidu Unlimited-OCR ([`pretok::pretokenize`]).
    DeepSeekV2,
    /// `Digits(individual_digits=true)` + `ByteLevel(use_regex=true)` (the
    /// GPT-2 word regex) — SmolLM2 / SmolVLM2 ([`pretok::pretokenize_smollm2`]).
    SmolLm2,
    /// The plain GPT-2 word regex, no Digits stage — OneChart / OPT over
    /// `vocab.json`+`merges.txt` ([`pretok::pretokenize_gpt2`]; selected by
    /// [`Tokenizer::from_opt_files`], never by `pre_tokenizer` detection —
    /// the slow-tokenizer files carry no such declaration).
    Gpt2,
}

/// Classify the `.pre_tokenizer` section into a supported [`PretokScheme`].
///
/// Fail-closed: an unrecognized declaration is a [`FocrError::FormatMismatch`]
/// (silently running the wrong pre-tokenizer would corrupt every downstream
/// id). A MISSING section maps to [`PretokScheme::DeepSeekV2`] — the historical
/// default, kept so the synthetic test fixtures (which carry no
/// `pre_tokenizer`) keep exercising the Baidu path.
fn classify_pretok(v: Option<&serde_json::Value>) -> FocrResult<PretokScheme> {
    let Some(v) = v else {
        return Ok(PretokScheme::DeepSeekV2);
    };
    let stages = v
        .get("pretokenizers")
        .and_then(|p| p.as_array())
        .ok_or_else(|| {
            FocrError::FormatMismatch(format!(
                "unsupported tokenizer.json pre_tokenizer (expected a Sequence): {v}"
            ))
        })?;
    fn ty(s: &serde_json::Value) -> &str {
        s.get("type").and_then(|t| t.as_str()).unwrap_or("")
    }
    // SmolLM2 / GPT-2: [Digits(individual_digits=true), ByteLevel(use_regex=true)].
    if stages.len() == 2
        && ty(&stages[0]) == "Digits"
        && stages[0].get("individual_digits").and_then(|b| b.as_bool()) == Some(true)
        && ty(&stages[1]) == "ByteLevel"
        && stages[1].get("use_regex").and_then(|b| b.as_bool()) == Some(true)
    {
        return Ok(PretokScheme::SmolLm2);
    }
    // DeepSeek-V2: [Split \p{N}{1,3}, Split CJK, Split word-regex,
    // ByteLevel(use_regex=false)] — anchor on the first Split's digit pattern
    // and the ByteLevel tail (the two load-bearing stages).
    if stages.len() == 4
        && ty(&stages[0]) == "Split"
        && stages[0].pointer("/pattern/Regex").and_then(|r| r.as_str()) == Some("\\p{N}{1,3}")
        && ty(&stages[3]) == "ByteLevel"
        && stages[3].get("use_regex").and_then(|b| b.as_bool()) == Some(false)
    {
        return Ok(PretokScheme::DeepSeekV2);
    }
    Err(FocrError::FormatMismatch(format!(
        "unsupported tokenizer.json pre_tokenizer sequence (neither the \
         DeepSeek-V2 4-stage Split nor the SmolLM2 Digits+ByteLevel shape): {v}"
    )))
}

// ── The tokenizer ───────────────────────────────────────────────────────────

/// The byte-level BPE tokenizer, loaded from a `tokenizer.json`.
///
/// Cheap to clone-by-reference (`&Tokenizer` everywhere); the big maps are owned
/// once. Construct with [`Tokenizer::from_file`] (or its alias
/// [`Tokenizer::load`]).
#[derive(Debug)]
pub struct Tokenizer {
    /// base-BPE token string (byte-level alphabet) → id.
    vocab: HashMap<String, u32>,
    /// id → token string, for decode. Covers base vocab AND added tokens.
    id_to_token: HashMap<u32, String>,
    /// merge rank: `(left, right)` → rank (lower = higher priority).
    merge_ranks: HashMap<(String, String), u32>,
    /// Added/special tokens, longest-content-first, for greedy left-to-right
    /// splitting of the input before BPE.
    added: Vec<AddedToken>,
    /// added-token content → id (exact-string match).
    added_by_content: HashMap<String, u32>,
    /// ids that are flagged `special` (for `skip_special_tokens` on decode).
    special_ids: std::collections::HashSet<u32>,
    /// The pre-tokenizer scheme the loaded `tokenizer.json` declared.
    scheme: PretokScheme,
}

/// An added token with its id and `special` flag.
#[derive(Debug, Clone)]
struct AddedToken {
    content: String,
    id: u32,
    #[allow(dead_code)]
    special: bool,
}

impl Tokenizer {
    /// Load the tokenizer from a `tokenizer.json` at `path`.
    ///
    /// The 9.9 MB file is fetched out-of-band and loaded lazily by path (never
    /// embedded). Parses `.model.vocab`, `.model.merges` (both encodings), and
    /// `.added_tokens`, then validates the pinned special-token ids (OQ-16 §6).
    ///
    /// # Errors
    /// * [`FocrError::ModelNotFound`] if the file can't be read.
    /// * [`FocrError::FormatMismatch`] if the JSON can't be parsed or a pinned
    ///   special-token id disagrees with the loaded table.
    pub fn from_file(path: &Path) -> FocrResult<Self> {
        let bytes = std::fs::read(path).map_err(|e| {
            FocrError::ModelNotFound(format!("tokenizer.json at {}: {e}", path.display()))
        })?;
        Self::from_json_bytes(&bytes)
    }

    /// Alias kept for call-site stability (older modules referenced `load`).
    ///
    /// # Errors
    /// See [`Tokenizer::from_file`].
    pub fn load(path: &Path) -> FocrResult<Self> {
        Self::from_file(path)
    }

    /// Build from in-memory `tokenizer.json` bytes (used by [`from_file`] and by
    /// tests with synthetic fixtures — no real 9.9 MB file required).
    ///
    /// # Errors
    /// [`FocrError::FormatMismatch`] on parse failure or pinned-id disagreement.
    pub fn from_json_bytes(bytes: &[u8]) -> FocrResult<Self> {
        let raw: RawTokenizer = serde_json::from_slice(bytes)
            .map_err(|e| FocrError::FormatMismatch(format!("tokenizer.json parse: {e}")))?;

        let scheme = classify_pretok(raw.pre_tokenizer.as_ref())?;
        let vocab = raw.model.vocab;

        let mut merge_ranks = HashMap::with_capacity(raw.model.merges.len());
        for (rank, m) in raw.model.merges.into_iter().enumerate() {
            if let Some(pair) = m.into_pair() {
                // First occurrence wins (merges are already in priority order);
                // duplicates are ignored to keep the earliest (best) rank.
                merge_ranks.entry(pair).or_insert(rank as u32);
            }
        }

        // Build the id→token map from the base vocab.
        let mut id_to_token: HashMap<u32, String> = HashMap::with_capacity(vocab.len());
        for (tok, &id) in &vocab {
            id_to_token.insert(id, tok.clone());
        }

        // Added tokens: seed the split table + content/id maps + decode strings.
        let mut added = Vec::with_capacity(raw.added_tokens.len());
        let mut added_by_content = HashMap::with_capacity(raw.added_tokens.len());
        let mut special_ids = std::collections::HashSet::new();
        for at in raw.added_tokens {
            // Added tokens decode to their literal content (NOT byte-level
            // remapped) — they are exact UTF-8 strings (OQ-16 §6).
            id_to_token.insert(at.id, at.content.clone());
            added_by_content.insert(at.content.clone(), at.id);
            if at.special {
                special_ids.insert(at.id);
            }
            added.push(AddedToken {
                content: at.content,
                id: at.id,
                special: at.special,
            });
        }
        // Longest content first → greedy split prefers the longest match (HF
        // `AddedVocabulary` is a longest-match trie; sorting by length desc and
        // scanning left-to-right reproduces it for our non-overlapping set).
        added.sort_by(|a, b| b.content.len().cmp(&a.content.len()).then(a.id.cmp(&b.id)));

        let tk = Tokenizer {
            vocab,
            id_to_token,
            merge_ranks,
            added,
            added_by_content,
            special_ids,
            scheme,
        };
        tk.validate_pinned_ids()?;
        Ok(tk)
    }

    /// Load the OneChart / OPT slow-tokenizer file triple (D9): `vocab.json`
    /// (token→id), `merges.txt` (`#version` header + one space-joined merge
    /// per line), and `added_tokens.json` (content→id; all treated as
    /// specials, HF `AddedVocabulary` semantics). Selects
    /// [`PretokScheme::Gpt2`] and validates the census §4/§7 pins.
    ///
    /// # Errors
    /// [`FocrError::ModelNotFound`] for missing files;
    /// [`FocrError::FormatMismatch`] on parse failure or a pin disagreement.
    pub fn from_opt_dir(dir: &Path) -> FocrResult<Self> {
        let read = |name: &str| -> FocrResult<Vec<u8>> {
            std::fs::read(dir.join(name)).map_err(|e| {
                FocrError::ModelNotFound(format!("{} at {}: {e}", name, dir.display()))
            })
        };
        Self::from_opt_files(
            &read("vocab.json")?,
            &read("merges.txt")?,
            &read("added_tokens.json")?,
        )
    }

    /// [`Tokenizer::from_opt_dir`] over in-memory bytes (tests / synthetic
    /// fixtures).
    ///
    /// # Errors
    /// [`FocrError::FormatMismatch`] on parse failure or pin disagreement.
    pub fn from_opt_files(
        vocab_json: &[u8],
        merges_txt: &[u8],
        added_tokens_json: &[u8],
    ) -> FocrResult<Self> {
        let vocab: HashMap<String, u32> = serde_json::from_slice(vocab_json)
            .map_err(|e| FocrError::FormatMismatch(format!("vocab.json parse: {e}")))?;
        let merges_text = std::str::from_utf8(merges_txt)
            .map_err(|e| FocrError::FormatMismatch(format!("merges.txt not UTF-8: {e}")))?;
        let mut merge_ranks = HashMap::new();
        let mut rank = 0u32;
        for line in merges_text.lines() {
            let line = line.trim_end();
            if line.is_empty() || line.starts_with("#version") {
                continue;
            }
            let Some((l, r)) = line.split_once(' ') else {
                return Err(FocrError::FormatMismatch(format!(
                    "merges.txt line without a space: {line:?}"
                )));
            };
            merge_ranks
                .entry((l.to_string(), r.to_string()))
                .or_insert(rank);
            rank += 1;
        }
        let added_map: HashMap<String, u32> = serde_json::from_slice(added_tokens_json)
            .map_err(|e| FocrError::FormatMismatch(format!("added_tokens.json parse: {e}")))?;

        let mut id_to_token: HashMap<u32, String> = HashMap::with_capacity(vocab.len());
        for (tok, &id) in &vocab {
            id_to_token.insert(id, tok.clone());
        }
        let mut added = Vec::with_capacity(added_map.len());
        let mut added_by_content = HashMap::with_capacity(added_map.len());
        let mut special_ids = std::collections::HashSet::new();
        for (content, id) in added_map {
            id_to_token.insert(id, content.clone());
            added_by_content.insert(content.clone(), id);
            special_ids.insert(id);
            added.push(AddedToken {
                content,
                id,
                special: true,
            });
        }
        // The OPT control tokens live in the BASE vocab but are splittable
        // specials (`special_tokens_map.json`: bos/eos/pad/unk) — the slow
        // `GPT2Tokenizer` resolves a literal `"</s>"` to id 2, never BPE.
        for surface in ["<s>", "<pad>", "</s>", "<unk>"] {
            if let Some(&id) = vocab.get(surface)
                && !added_by_content.contains_key(surface)
            {
                added_by_content.insert(surface.to_string(), id);
                special_ids.insert(id);
                added.push(AddedToken {
                    content: surface.to_string(),
                    id,
                    special: true,
                });
            }
        }
        added.sort_by(|a, b| b.content.len().cmp(&a.content.len()).then(a.id.cmp(&b.id)));

        let tk = Tokenizer {
            vocab,
            id_to_token,
            merge_ranks,
            added,
            added_by_content,
            special_ids,
            scheme: PretokScheme::Gpt2,
        };
        tk.validate_pinned_ids()?;
        Ok(tk)
    }

    /// Cross-check the pinned special-token ids against the loaded added-token
    /// table (per scheme: OQ-16 §6 for Baidu, spec §5 for SmolLM2, the
    /// OneChart census §4/§7 for OPT). A disagreement means the wrong
    /// tokenizer files were supplied and every downstream id would be wrong —
    /// fail loud.
    fn validate_pinned_ids(&self) -> FocrResult<()> {
        let checks: &[(&str, u32)] = match self.scheme {
            PretokScheme::DeepSeekV2 => &[
                ("<image>", special::IMAGE),
                ("<|ref|>", special::REF),
                ("<|/ref|>", special::REF_END),
                ("<|det|>", special::DET),
                ("<|/det|>", special::DET_END),
                ("<|grounding|>", special::GROUNDING),
                ("<|User|>", special::USER),
                ("<|Assistant|>", special::ASSISTANT),
            ],
            PretokScheme::SmolLm2 => &[
                ("<|endoftext|>", special_smollm2::UNK),
                ("<|im_start|>", special_smollm2::BOS),
                ("<|im_end|>", special_smollm2::PAD),
                ("<global-img>", special_smollm2::GLOBAL_IMG),
                (
                    "<fake_token_around_image>",
                    special_smollm2::FAKE_AROUND_IMAGE,
                ),
                ("<image>", special_smollm2::IMAGE),
                ("<end_of_utterance>", special_smollm2::END_OF_UTTERANCE),
            ],
            PretokScheme::Gpt2 => &[
                ("<imgpad>", special_opt::IMG_PAD),
                ("<img>", special_opt::IMG_START),
                ("</img>", special_opt::IMG_END),
                ("<Number>", special_opt::NUMBER),
            ],
        };
        for &(content, want) in checks {
            // Only validate tokens that the supplied file actually declares; a
            // tiny synthetic fixture (tests) need not carry the full OCR table.
            if let Some(&got) = self.added_by_content.get(content)
                && got != want
            {
                return Err(FocrError::FormatMismatch(format!(
                    "tokenizer.json id mismatch for {content}: file says {got}, expected {want}"
                )));
            }
        }
        Ok(())
    }

    /// Encode `text` to token ids, **without** adding any special tokens
    /// (`add_special_tokens=False`, OQ-16 §5). Added/special tokens that appear
    /// literally in `text` are still recognized and emitted as their single id
    /// (e.g. a literal `"<image>"` → one id-128815 token).
    ///
    /// # Errors
    /// Never fails for valid UTF-8 input: byte-level coverage guarantees every
    /// byte maps to a known vocab symbol, so there is no UNK path. Returns
    /// [`FocrError::FormatMismatch`] only if the loaded vocab is missing a
    /// single-byte symbol (a corrupt `tokenizer.json`).
    pub fn encode(&self, text: &str) -> FocrResult<Vec<u32>> {
        let mut ids = Vec::new();
        // Split the text on added/special tokens first (HF AddedVocabulary).
        for segment in self.split_on_added(text) {
            match segment {
                Segment::Added(id) => ids.push(id),
                Segment::Text(s) => self.encode_text_segment(s, &mut ids)?,
            }
        }
        Ok(ids)
    }

    /// BPE-encode a plain text segment (no added tokens inside) and append ids.
    fn encode_text_segment(&self, text: &str, out: &mut Vec<u32>) -> FocrResult<()> {
        let pieces = match self.scheme {
            PretokScheme::DeepSeekV2 => pretok::pretokenize(text),
            PretokScheme::SmolLm2 => pretok::pretokenize_smollm2(text),
            PretokScheme::Gpt2 => pretok::pretokenize_gpt2(text),
        };
        for piece in pieces {
            // `piece` is already byte-level remapped → its chars are vocab
            // symbols. Apply BPE merges, then map merged symbols to ids.
            let symbols = self.bpe(&piece);
            for sym in symbols {
                let id = self.vocab.get(&sym).copied().ok_or_else(|| {
                    FocrError::FormatMismatch(format!(
                        "byte-level symbol {sym:?} missing from vocab (corrupt tokenizer.json)"
                    ))
                })?;
                out.push(id);
            }
        }
        Ok(())
    }

    /// The core BPE merge loop over one pre-tokenized (byte-level) `piece`.
    /// Returns the final list of merged symbol strings. Greedily applies the
    /// lowest-rank (highest-priority) adjacent merge until none apply — the HF
    /// `BPE` word-merge algorithm.
    fn bpe(&self, piece: &str) -> Vec<String> {
        // Start: one symbol per byte-level char.
        let mut symbols: Vec<String> = piece.chars().map(|c| c.to_string()).collect();
        if symbols.len() < 2 {
            return symbols;
        }
        loop {
            // Find the adjacent pair with the lowest merge rank.
            let mut best: Option<(usize, u32)> = None;
            for i in 0..symbols.len() - 1 {
                let pair = (symbols[i].clone(), symbols[i + 1].clone());
                if let Some(&rank) = self.merge_ranks.get(&pair) {
                    match best {
                        Some((_, br)) if rank >= br => {}
                        _ => best = Some((i, rank)),
                    }
                }
            }
            let Some((i, _)) = best else { break };
            // Merge symbols[i] and symbols[i+1].
            let merged = format!("{}{}", symbols[i], symbols[i + 1]);
            symbols.splice(i..=i + 1, std::iter::once(merged));
            if symbols.len() < 2 {
                break;
            }
        }
        symbols
    }

    /// Split `text` into a sequence of added-token ids and plain-text runs,
    /// scanning left-to-right and greedily preferring the longest added-token
    /// content at each position (HF `AddedVocabulary`). The `added` table is
    /// pre-sorted longest-first so the first content that matches at a position
    /// is the longest.
    fn split_on_added<'a>(&self, text: &'a str) -> Vec<Segment<'a>> {
        if self.added.is_empty() {
            return vec![Segment::Text(text)];
        }
        let bytes = text.as_bytes();
        let mut out = Vec::new();
        let mut run_start = 0usize; // start of the current plain-text run
        let mut i = 0usize;
        while i < bytes.len() {
            let mut matched = None;
            for at in &self.added {
                let c = at.content.as_bytes();
                if !c.is_empty() && bytes[i..].starts_with(c) {
                    matched = Some((c.len(), at.id));
                    break; // longest-first ordering → first hit is longest
                }
            }
            if let Some((len, id)) = matched {
                if run_start < i {
                    out.push(Segment::Text(&text[run_start..i]));
                }
                out.push(Segment::Added(id));
                i += len;
                run_start = i;
            } else {
                // advance by one full char (stay on UTF-8 boundaries)
                let ch_len = utf8_char_len(bytes[i]);
                i += ch_len;
            }
        }
        if run_start < text.len() {
            out.push(Segment::Text(&text[run_start..]));
        }
        out
    }

    /// Decode token ids back to a `String` ([SPEC-110]). Base-vocab tokens are
    /// byte-level symbols that are un-mapped back to raw bytes and UTF-8 decoded;
    /// added/special tokens contribute their literal content. By default special
    /// tokens ARE included (the model's own decode keeps structural glyphs); use
    /// [`Tokenizer::decode_skip_special`] to drop them.
    ///
    /// # Errors
    /// [`FocrError::FormatMismatch`] if an id is unknown or the reconstructed
    /// bytes are not valid UTF-8 (a truncated multi-byte sequence at the tail is
    /// reported rather than silently lossily replaced).
    pub fn decode(&self, ids: &[u32]) -> FocrResult<String> {
        self.decode_inner(ids, false)
    }

    /// Decode, skipping tokens flagged `special:true` (OQ-16 §6) — the
    /// `skip_special_tokens=True` path used when emitting clean markdown.
    ///
    /// # Errors
    /// See [`Tokenizer::decode`].
    pub fn decode_skip_special(&self, ids: &[u32]) -> FocrResult<String> {
        self.decode_inner(ids, true)
    }

    fn decode_inner(&self, ids: &[u32], skip_special: bool) -> FocrResult<String> {
        // Accumulate raw bytes: base tokens contribute byte-level-decoded bytes,
        // added tokens contribute their literal UTF-8 bytes. We flush to a single
        // UTF-8 string at the end so multi-byte chars split across base tokens
        // reassemble correctly.
        let mut bytes: Vec<u8> = Vec::new();
        for &id in ids {
            if skip_special && self.special_ids.contains(&id) {
                continue;
            }
            self.token_bytes(id, &mut bytes)?;
        }
        String::from_utf8(bytes)
            .map_err(|e| FocrError::FormatMismatch(format!("decode: invalid UTF-8: {e}")))
    }

    /// Append token `id`'s bytes to `out` and return whether it is an added
    /// token. An added token contributes its literal content (NOT byte-level
    /// remapped); a base token's chars are byte-level symbols, mapped back.
    /// On error `out` is left as it was.
    fn token_bytes(&self, id: u32, out: &mut Vec<u8>) -> FocrResult<bool> {
        let tok = self
            .id_to_token
            .get(&id)
            .ok_or_else(|| FocrError::FormatMismatch(format!("decode: unknown token id {id}")))?;
        if self.added_by_content.contains_key(tok) {
            out.extend_from_slice(tok.as_bytes());
            return Ok(true);
        }
        let start = out.len();
        for c in tok.chars() {
            let Some(b) = pretok::char_to_byte(c) else {
                out.truncate(start);
                return Err(FocrError::FormatMismatch(format!(
                    "decode: token id {id} has non-byte-level symbol {c:?}"
                )));
            };
            out.push(b);
        }
        Ok(false)
    }

    /// The id of an added/special token by its literal content, if present.
    pub fn token_to_id(&self, content: &str) -> Option<u32> {
        self.added_by_content
            .get(content)
            .copied()
            .or_else(|| self.vocab.get(content).copied())
    }

    /// The literal string for an id (added content or byte-level symbol), if
    /// present. (Byte-level symbols are returned un-decoded — for human display
    /// prefer [`Tokenizer::decode`].)
    pub fn id_to_token(&self, id: u32) -> Option<&str> {
        self.id_to_token.get(&id).map(String::as_str)
    }

    /// Number of base-vocab entries (excludes added tokens). The embedding /
    /// LM-head shape is the model's `vocab_size` (129280), NOT this — see
    /// OQ-16 §7.
    pub fn vocab_size(&self) -> usize {
        self.vocab.len()
    }

    /// The pre-tokenizer scheme the loaded `tokenizer.json` declared.
    pub fn scheme(&self) -> PretokScheme {
        self.scheme
    }

    /// BOS id — Baidu `<｜begin▁of▁sentence｜>` (0, OQ-16 §5) or SmolLM2
    /// `<|im_start|>` (1). Prompt builders own their own framing; nothing is
    /// auto-prepended.
    pub fn bos_id(&self) -> u32 {
        match self.scheme {
            PretokScheme::DeepSeekV2 => special::BOS,
            PretokScheme::SmolLm2 => special_smollm2::BOS,
            PretokScheme::Gpt2 => special_opt::BOS_EOS,
        }
    }
    /// EOS / generation-stop id — Baidu `<｜end▁of▁sentence｜>` (1) or SmolVLM2
    /// `<end_of_utterance>` (49279). Never auto-appended.
    pub fn eos_id(&self) -> u32 {
        match self.scheme {
            PretokScheme::DeepSeekV2 => special::EOS,
            PretokScheme::SmolLm2 => special_smollm2::END_OF_UTTERANCE,
            PretokScheme::Gpt2 => special_opt::BOS_EOS,
        }
    }
    /// PAD id — Baidu `<｜▁pad▁｜>` (2) or SmolLM2 `<|im_end|>` (2).
    pub fn pad_id(&self) -> u32 {
        match self.scheme {
            PretokScheme::DeepSeekV2 => special::PAD,
            PretokScheme::SmolLm2 => special_smollm2::PAD,
            PretokScheme::Gpt2 => special_opt::PAD,
        }
    }
    /// `<image>` splice-slot id — Baidu 128815 ([SPEC-035]) or SmolVLM2 49190
    /// (64 slots per 512² frame, spec §5).
    pub fn image_id(&self) -> u32 {
        match self.scheme {
            PretokScheme::DeepSeekV2 => special::IMAGE,
            PretokScheme::SmolLm2 => special_smollm2::IMAGE,
            PretokScheme::Gpt2 => special_opt::IMG_PAD,
        }
    }
}

/// Incremental, UTF-8-safe detokenizing for streaming (U2-X1). Byte-level BPE
/// can split one character across several tokens, so a prefix of the ids may
/// end inside a character, which [`Tokenizer::decode`] reports as an error.
/// This decoder holds an incomplete trailing sequence back until a later token
/// completes it, so it only ever emits whole characters.
///
/// An added token (`<image>`, v2's `<PAGE>`, …) ends a byte segment: the
/// pending bytes are flushed first, as HF's ByteLevel decoder
/// (`errors="replace"`) decodes each segment between added tokens on its own.
/// Invalid bytes become U+FFFD wherever `String::from_utf8_lossy` would put
/// them. So the text pushed, plus [`StreamDecoder::flush`] at the end, is the
/// per-segment lossy decode of the ids; when the ids decode as valid UTF-8 it
/// is exactly [`Tokenizer::decode`].
#[derive(Debug, Clone, Default)]
pub struct StreamDecoder {
    /// Bytes of the current segment not yet emitted: an incomplete trailing
    /// character, at most 3 bytes between pushes.
    pending: Vec<u8>,
}

impl StreamDecoder {
    /// An empty decoder.
    #[must_use]
    pub fn new() -> Self {
        Self::default()
    }

    /// Decode token `id` and append the whole characters it completes to
    /// `out`.
    ///
    /// # Errors
    /// [`FocrError::FormatMismatch`] for an id the tokenizer does not know or a
    /// base token with a non-byte-level symbol, as [`Tokenizer::decode`]; the
    /// decoder is then unchanged.
    pub fn push(&mut self, tok: &Tokenizer, id: u32, out: &mut String) -> FocrResult<()> {
        let start = self.pending.len();
        if tok.token_bytes(id, &mut self.pending)? {
            let literal = self.pending.split_off(start);
            self.flush(out);
            out.push_str(&String::from_utf8_lossy(&literal));
        } else {
            self.emit_complete(out);
        }
        Ok(())
    }

    /// End of the stream or of a page segment: append what is held back, an
    /// incomplete character as U+FFFD.
    pub fn flush(&mut self, out: &mut String) {
        out.push_str(&String::from_utf8_lossy(&self.pending));
        self.pending.clear();
    }

    /// Emit every complete character and every invalid sequence (as U+FFFD),
    /// keeping only an incomplete trailing sequence a later byte could finish.
    fn emit_complete(&mut self, out: &mut String) {
        let mut at = 0;
        while at < self.pending.len() {
            match std::str::from_utf8(&self.pending[at..]) {
                Ok(text) => {
                    out.push_str(text);
                    at = self.pending.len();
                }
                Err(e) => {
                    let valid = at + e.valid_up_to();
                    out.push_str(&String::from_utf8_lossy(&self.pending[at..valid]));
                    let Some(bad) = e.error_len() else {
                        at = valid;
                        break;
                    };
                    out.push(char::REPLACEMENT_CHARACTER);
                    at = valid + bad;
                }
            }
        }
        self.pending.drain(..at);
    }
}

/// A run produced by [`Tokenizer::split_on_added`].
enum Segment<'a> {
    /// A plain-text run to BPE-encode.
    Text(&'a str),
    /// An added/special token already resolved to its id.
    Added(u32),
}

/// Length in bytes of a UTF-8 char from its lead byte (1..=4). Used to advance
/// the added-token scanner on char boundaries without re-decoding. Shared with
/// the sibling [`tiktoken`] module's special-token scanner.
pub(super) fn utf8_char_len(lead: u8) -> usize {
    if lead < 0x80 {
        1
    } else if lead >> 5 == 0b110 {
        2
    } else if lead >> 4 == 0b1110 {
        3
    } else if lead >> 3 == 0b11110 {
        4
    } else {
        1 // continuation/invalid byte — advance one to make progress
    }
}

/// The exact Python snippet that regenerates [`unicode_tables`]. Kept here (not
/// runnable Rust) so a maintainer can reproduce the tables bit-for-bit and a
/// batch-verify can reconcile the UCD version (see [`unicode_tables::UCD_VERSION`]).
///
/// ```text
/// import unicodedata
/// for cat in 'LMNPS':
///     ranges, start, prev = [], None, None
///     for cp in range(0x110000):
///         hit = unicodedata.category(chr(cp))[0] == cat
///         if hit:
///             if start is None: start = prev = cp
///             elif cp == prev + 1: prev = cp
///             else: ranges.append((start, prev)); start = prev = cp
///         elif start is not None:
///             ranges.append((start, prev)); start = prev = None
///     if start is not None: ranges.append((start, prev))
///     # emit `pub static {CAT}: &[(u32,u32)] = &[ (lo,hi), … ];`
/// ```
#[allow(dead_code)]
const UNICODE_TABLE_REGEN: () = ();

#[cfg(test)]
pub(crate) mod tests {
    use super::*;

    /// A tiny synthetic `tokenizer.json` (no real 9.9 MB file). The base vocab
    /// covers the byte-level symbols for the lowercase letters used in tests
    /// plus the space glyph `Ġ`, and a couple of merges so we can prove merge
    /// ordering. Added tokens carry the pinned `<image>` id to exercise the
    /// validation + splitting paths. `pub(super)` so the [`super::ops`]
    /// trait-level tests run BOTH impls over the same fixtures (no drift).
    pub(super) fn tiny_json() -> String {
        // Byte-level: 'a'..'z' map to themselves; space → "Ġ".
        // vocab ids are arbitrary (we only test id-EXACTNESS of merge ordering
        // and round-trip, not against HF here).
        // Symbols present: a b c d e f Ġ ab abc Ġa  (merge "a"+"b"->"ab",
        // "ab"+"c"->"abc"; lower rank = applied first).
        r#"{
          "version": "1.0",
          "added_tokens": [
            {"id": 128815, "content": "<image>", "special": true},
            {"id": 100,    "content": "<|x|>",   "special": false}
          ],
          "normalizer": {"type":"Sequence","normalizers":[]},
          "model": {
            "type": "BPE",
            "vocab": {
              "a": 0, "b": 1, "c": 2, "d": 3, "e": 4, "f": 5,
              "Ġ": 6, "ab": 7, "abc": 8, "Ġa": 9, "Ġd": 10
            },
            "merges": [
              ["a", "b"],
              ["ab", "c"],
              ["Ġ", "a"],
              ["Ġ", "d"]
            ]
          }
        }"#
        .to_string()
    }

    fn tk() -> Tokenizer {
        Tokenizer::from_json_bytes(tiny_json().as_bytes()).expect("tiny tokenizer loads")
    }

    #[test]
    fn loads_and_validates_pinned_ids() {
        let t = tk();
        assert_eq!(t.token_to_id("<image>"), Some(special::IMAGE));
        assert_eq!(t.image_id(), 128815);
        assert_eq!(t.bos_id(), 0);
        assert_eq!(t.eos_id(), 1);
        assert_eq!(t.vocab_size(), 11);
    }

    #[test]
    fn pinned_id_mismatch_is_format_error() {
        // Same fixture but with the wrong <image> id → must reject.
        let bad = tiny_json().replace("\"id\": 128815", "\"id\": 999");
        let err = Tokenizer::from_json_bytes(bad.as_bytes()).unwrap_err();
        assert!(matches!(err, FocrError::FormatMismatch(_)), "got {err:?}");
    }

    #[test]
    fn bpe_merge_ordering() {
        let t = tk();
        // "abc" → merges: "a"+"b"->"ab" (rank0), "ab"+"c"->"abc" (rank1).
        // Final single symbol "abc" = id 8.
        assert_eq!(t.encode("abc").unwrap(), vec![8]);
        // "ab" stops at "ab" = id 7.
        assert_eq!(t.encode("ab").unwrap(), vec![7]);
        // "ba" has no merge → ['b','a'] = [1,0].
        assert_eq!(t.encode("ba").unwrap(), vec![1, 0]);
    }

    #[test]
    fn merge_priority_is_rank_not_position() {
        let t = tk();
        // " a" pretokenizes to "Ġa" (leading space + letter). Merge "Ġ"+"a"
        // (rank 2) → "Ġa" = id 9.
        assert_eq!(t.encode(" a").unwrap(), vec![9]);
    }

    #[test]
    fn special_token_splitting() {
        let t = tk();
        // "ab<image>c" → BPE("ab")=[7], <image>=128815, BPE("c")=[2].
        assert_eq!(t.encode("ab<image>c").unwrap(), vec![7, 128815, 2]);
        // A non-special added token still splits.
        assert_eq!(t.encode("a<|x|>b").unwrap(), vec![0, 100, 1]);
        // Longest-match: ensure "<image>" wins as a whole (no partial BPE of '<').
        let ids = t.encode("<image>").unwrap();
        assert_eq!(ids, vec![128815]);
    }

    #[test]
    fn round_trip_encode_decode() {
        let t = tk();
        // Base-token round trip.
        let ids = t.encode("abc").unwrap();
        assert_eq!(t.decode(&ids).unwrap(), "abc");
        // With a space (byte-level Ġ must invert back to ' ').
        let ids2 = t.encode(" a").unwrap();
        assert_eq!(t.decode(&ids2).unwrap(), " a");
        // Mixed with an added token: decode includes the literal content.
        let ids3 = t.encode("ab<image>c").unwrap();
        assert_eq!(t.decode(&ids3).unwrap(), "ab<image>c");
    }

    #[test]
    fn decode_skip_special_drops_specials() {
        let t = tk();
        let ids = t.encode("ab<image>c").unwrap();
        // <image> is special:true → dropped; <|x|> would be kept (special:false).
        assert_eq!(t.decode_skip_special(&ids).unwrap(), "abc");
        let ids2 = t.encode("a<|x|>b").unwrap();
        assert_eq!(t.decode_skip_special(&ids2).unwrap(), "a<|x|>b");
    }

    #[test]
    fn byte_level_non_ascii_round_trips_through_bytes() {
        // 'é' = U+00E9 → bytes C3 A9 → byte-level chars 'Ã','©'. Neither is in
        // our tiny vocab, so encode would error — but decode of a synthetic
        // byte-split must reassemble. Use a vocab that has the two symbols.
        let json = r#"{
          "added_tokens": [],
          "model": {"type":"BPE",
            "vocab": {"Ã": 0, "©": 1},
            "merges": []
          }
        }"#;
        let t = Tokenizer::from_json_bytes(json.as_bytes()).unwrap();
        let ids = t.encode("é").unwrap(); // two byte-level symbols
        assert_eq!(ids, vec![0, 1]);
        // Decode reassembles the two bytes into the original 'é'.
        assert_eq!(t.decode(&ids).unwrap(), "é");
    }

    #[test]
    fn legacy_joined_merge_format_is_accepted() {
        // Old `tokenizer.json` encodes merges as space-joined strings.
        let json = r#"{
          "added_tokens": [],
          "model": {"type":"BPE",
            "vocab": {"a":0,"b":1,"ab":2},
            "merges": ["a b"]
          }
        }"#;
        let t = Tokenizer::from_json_bytes(json.as_bytes()).unwrap();
        assert_eq!(t.encode("ab").unwrap(), vec![2]);
    }

    #[test]
    fn decode_unknown_id_errors() {
        let t = tk();
        let err = t.decode(&[424242]).unwrap_err();
        assert!(matches!(err, FocrError::FormatMismatch(_)), "got {err:?}");
    }

    #[test]
    fn from_file_missing_is_model_not_found() {
        let err = Tokenizer::from_file(Path::new("/nonexistent/tokenizer.json")).unwrap_err();
        assert!(matches!(err, FocrError::ModelNotFound(_)), "got {err:?}");
    }

    #[test]
    fn empty_input_encodes_empty() {
        let t = tk();
        assert_eq!(t.encode("").unwrap(), Vec::<u32>::new());
        assert_eq!(t.decode(&[]).unwrap(), "");
    }

    // ── real-vocab conformance (env-gated on the ~9.9 MB tokenizer.json) ─────
    // FOCR_TOKENIZER_JSON points at the pinned file; else the truth-pack
    // snapshot path (gitignored, fetched out-of-band by scripts/fetch_sources.sh);
    // absent ⇒ skip (the model-gated pattern, matching tiktoken.rs).

    // (`pub(super)` so the [`super::ops`] trait tests reuse the same gate.)
    pub(super) fn load_real() -> Option<Tokenizer> {
        let path = std::env::var("FOCR_TOKENIZER_JSON").unwrap_or_else(|_| {
            concat!(
                env!("CARGO_MANIFEST_DIR"),
                "/docs/truth-pack/snapshots/tokenizer.json"
            )
            .to_string()
        });
        let path = Path::new(&path);
        if !path.is_file() {
            eprintln!(
                "SKIP baidu tokenizer conformance: {} absent \
                 (scripts/fetch_sources.sh or FOCR_TOKENIZER_JSON)",
                path.display()
            );
            return None;
        }
        Some(Tokenizer::from_file(path).expect("pinned tokenizer.json must load"))
    }

    #[test]
    fn real_vocab_anchors() {
        let Some(t) = load_real() else {
            return;
        };
        assert_eq!(t.vocab_size(), 128000);
        // The runtime-hardcoded specials must each encode to their single id.
        assert_eq!(
            t.encode("<｜begin▁of▁sentence｜>").unwrap(),
            vec![special::BOS]
        );
        assert_eq!(
            t.encode("<｜end▁of▁sentence｜>").unwrap(),
            vec![special::EOS]
        );
        assert_eq!(t.encode("<｜▁pad▁｜>").unwrap(), vec![special::PAD]);
        assert_eq!(t.encode("<image>").unwrap(), vec![special::IMAGE]);
        assert_eq!(t.encode("<|grounding|>").unwrap(), vec![special::GROUNDING]);
        // ASCII-pipe role glyphs are distinct from the fullwidth-bar DeepSeek
        // glyphs (the glyph-vs-ASCII distinction is load-bearing, OQ-16 §6).
        assert_eq!(t.encode("<|User|>").unwrap(), vec![special::USER]);
        assert_eq!(t.encode("<|Assistant|>").unwrap(), vec![special::ASSISTANT]);
        assert_ne!(t.encode("<｜User｜>").unwrap(), vec![special::USER]);
    }

    /// Hold `t` to every record of a token-id golden: `encode(text) == ids`,
    /// `decode(ids) == decoded`, and, when the record carries it,
    /// `decode_skip_special(ids) == decoded_skip_special`. Each mismatch is
    /// printed; the count is returned.
    fn fixture_mismatches(t: &Tokenizer, cases: &[serde_json::Value]) -> usize {
        let mut mismatches = 0usize;
        for rec in cases {
            let text = rec["text"].as_str().unwrap();
            let want: Vec<u32> = rec["ids"]
                .as_array()
                .unwrap()
                .iter()
                .map(|x| x.as_u64().unwrap() as u32)
                .collect();
            let got = t.encode(text).unwrap();
            if got != want {
                let pos = got
                    .iter()
                    .zip(&want)
                    .position(|(a, b)| a != b)
                    .unwrap_or_else(|| got.len().min(want.len()));
                eprintln!(
                    "ENC MISMATCH {{\"case\": {text:?}, \"len\": {}, \"mismatch_pos\": {pos}}}\n  \
                     got  {got:?}\n  want {want:?}",
                    want.len()
                );
                mismatches += 1;
            }
            // Decode direction: EXACTLY the reference ids back to the reference
            // string (skip_special_tokens=false — literal round-trip).
            let want_decoded = rec["decoded"].as_str().unwrap();
            let got_decoded = t.decode(&want).unwrap();
            if got_decoded != want_decoded {
                eprintln!(
                    "DEC MISMATCH {{\"case\": {text:?}, \"mismatch_pos\": \"none\"}}\n  \
                     got  {got_decoded:?}\n  want {want_decoded:?}"
                );
                mismatches += 1;
            }
            if let Some(want_skipped) = rec["decoded_skip_special"].as_str() {
                let got_skipped = t.decode_skip_special(&want).unwrap();
                if got_skipped != want_skipped {
                    eprintln!(
                        "SKIP-SPECIAL DEC MISMATCH {{\"case\": {text:?}}}\n  \
                         got  {got_skipped:?}\n  want {want_skipped:?}"
                    );
                    mismatches += 1;
                }
            }
        }
        mismatches
    }

    /// **L0/L4 — the Baidu tokenizer token-id-EXACT conformance gate (OQ-16,
    /// bd-re8.8).** Parses the committed golden fixtures — generated by the
    /// reference HF `tokenizers` engine (the exact Rust crate
    /// `LlamaTokenizerFast` wraps) over the pinned `tokenizer.json` via
    /// `scripts/gen_token_id_fixtures.py` — and asserts our encoder reproduces
    /// every id stream AND our decoder every decoded string exactly. No
    /// decoder/vision parity bead may close while this is red (AGENTS.md
    /// doctrine).
    #[test]
    fn baidu_token_id_conformance_gate() {
        let Some(t) = load_real() else {
            return;
        };
        const EXPECTED: &str = include_str!(concat!(
            env!("CARGO_MANIFEST_DIR"),
            "/tests/fixtures/tokenizer_baidu/expected.json"
        ));
        let v: serde_json::Value = serde_json::from_str(EXPECTED).unwrap();
        // The fixture must match the tokenizer.json it was generated from —
        // ids from a different serialization are not comparable.
        assert_eq!(
            v["_meta"]["tokenizer_json_sha256"].as_str().unwrap(),
            "a02f8fd5228c90256bb4f6554c34a579d48f909e5beb232dc4afad870b55a8b4",
            "fixture was generated against a different tokenizer.json pin"
        );
        let cases = v["fixtures"].as_array().expect("fixtures array");
        let num_cases = v["_meta"]["num_cases"].as_u64().unwrap() as usize;
        assert_eq!(cases.len(), num_cases, "fixture _meta.num_cases drift");
        assert!(
            cases.len() >= 100,
            "conformance corpus must stay >= 100 cases"
        );
        let mismatches = fixture_mismatches(&t, cases);
        assert_eq!(
            mismatches, 0,
            "tok_id_mismatch_count must be 0 (got {mismatches})"
        );
    }

    // ── StreamDecoder (U2-X1) ───────────────────────────────────────────────

    const PAGE: u32 = special::PAGE;
    /// Multi-byte base tokens of [`byte_tokenizer`], by id: whole "中", its
    /// first two bytes, its last byte + "a" (crossing a character boundary),
    /// 🦀 as two halves, whole "é".
    const MULTI: [(u32, &[u8]); 6] = [
        (2000, "中".as_bytes()),
        (2001, &[0xE4, 0xB8]),
        (2002, &[0xAD, b'a']),
        (2003, &[0xF0, 0x9F]),
        (2004, &[0xA6, 0x80]),
        (2005, "é".as_bytes()),
    ];

    /// The byte-level alphabet symbol of every byte.
    fn byte_symbols() -> [char; 256] {
        let mut sym = ['\0'; 256];
        for c in (0..0x144).filter_map(char::from_u32) {
            if let Some(b) = pretok::char_to_byte(c) {
                sym[b as usize] = c;
            }
        }
        sym
    }

    /// The synthetic byte tokenizer's id for byte `b` (also used by the
    /// postprocess page-split tests).
    pub(crate) fn byte_id(b: u8) -> u32 {
        1000 + u32::from(b)
    }

    /// A synthetic byte-level tokenizer: every byte as its own token (id 1000 +
    /// byte), the [`MULTI`] tokens, and the added tokens `<image>` and `<PAGE>`.
    pub(crate) fn byte_tokenizer() -> Tokenizer {
        byte_tokenizer_with(&[(special::IMAGE, "<image>", true), (PAGE, "<PAGE>", false)])
    }

    /// [`byte_tokenizer`] with `added` as its `(id, content, special)` added
    /// tokens.
    fn byte_tokenizer_with(added: &[(u32, &str, bool)]) -> Tokenizer {
        let sym = byte_symbols();
        let spell = |bytes: &[u8]| bytes.iter().map(|&b| sym[b as usize]).collect::<String>();
        let mut vocab = serde_json::Map::new();
        for b in 0..=255u8 {
            vocab.insert(spell(&[b]), byte_id(b).into());
        }
        for (id, bytes) in MULTI {
            vocab.insert(spell(bytes), id.into());
        }
        let added: Vec<serde_json::Value> = added
            .iter()
            .map(|&(id, content, special)| {
                serde_json::json!({"id": id, "content": content, "special": special})
            })
            .collect();
        let json = serde_json::json!({
            "version": "1.0",
            "added_tokens": added,
            "normalizer": {"type": "Sequence", "normalizers": []},
            "model": {"type": "BPE", "vocab": vocab, "merges": []}
        });
        Tokenizer::from_json_bytes(json.to_string().as_bytes()).expect("byte tokenizer loads")
    }

    fn byte_ids(text: &str) -> Vec<u32> {
        text.bytes().map(byte_id).collect()
    }

    /// The oracle: each run of base tokens between added tokens decoded with
    /// `from_utf8_lossy` on its own, added tokens as their literal content.
    fn lossy_segments(t: &Tokenizer, ids: &[u32]) -> String {
        let (mut out, mut seg) = (String::new(), Vec::new());
        for &id in ids {
            let piece = t.id_to_token(id).expect("known id");
            if t.added_by_content.contains_key(piece) {
                out.push_str(&String::from_utf8_lossy(&seg));
                seg.clear();
                out.push_str(piece);
            } else {
                seg.extend(piece.chars().map(|c| pretok::char_to_byte(c).unwrap()));
            }
        }
        out.push_str(&String::from_utf8_lossy(&seg));
        out
    }

    /// Push `ids` one at a time; return what each push emitted and the flush.
    fn stream(t: &Tokenizer, ids: &[u32]) -> (Vec<String>, String) {
        let mut d = StreamDecoder::new();
        let emits = ids
            .iter()
            .map(|&id| {
                let mut out = String::new();
                d.push(t, id, &mut out).unwrap();
                out
            })
            .collect();
        let mut tail = String::new();
        d.flush(&mut tail);
        (emits, tail)
    }

    #[test]
    fn stream_decoder_emits_split_characters_once_complete() {
        let t = byte_tokenizer();
        let cases: [(&str, Vec<u32>, &[&str]); 6] = [
            ("中 as 3 byte tokens", byte_ids("中"), &["", "", "中"]),
            ("🦀 as 4 byte tokens", byte_ids("🦀"), &["", "", "", "🦀"]),
            ("🦀 as 2 halves", vec![2003, 2004], &["", "🦀"]),
            ("a token across a boundary", vec![2001, 2002], &["", "中a"]),
            (
                "whole tokens",
                vec![2000, 2005, byte_id(b'x')],
                &["中", "é", "x"],
            ),
            (
                "split, then ASCII, then split",
                [
                    byte_ids("é")[..1].to_vec(),
                    vec![byte_id(0xA9), byte_id(b'!')],
                    byte_ids("中"),
                ]
                .concat(),
                &["", "é", "!", "", "", "中"],
            ),
        ];
        for (what, ids, want) in cases {
            let (emits, tail) = stream(&t, &ids);
            assert_eq!(emits, want, "{what}");
            assert_eq!(tail, "", "{what}");
            assert_eq!(emits.concat(), t.decode(&ids).unwrap(), "{what}");
        }
    }

    #[test]
    fn stream_decoder_ends_a_segment_at_an_added_token() {
        let t = byte_tokenizer();
        // The first byte of 中, then <PAGE>, then text: the partial character
        // becomes U+FFFD at the end of the first segment and is not carried on.
        let ids = [byte_id(0xE4), PAGE, byte_id(b'a'), 2000];
        let (emits, tail) = stream(&t, &ids);
        assert_eq!(emits, ["", "\u{FFFD}<PAGE>", "a", "中"]);
        assert_eq!(tail, "");
        assert_eq!(emits.concat(), lossy_segments(&t, &ids));
        assert!(t.decode(&ids).is_err(), "the strict decode rejects it");
        // Invalid bytes inside a segment: U+FFFD where from_utf8_lossy puts it.
        let ids = [byte_id(0xFF), byte_id(b'a'), byte_id(0xE4), byte_id(b'b')];
        let (emits, _) = stream(&t, &ids);
        assert_eq!(emits, ["\u{FFFD}", "a", "", "\u{FFFD}b"]);
        // A partial character at the end of the stream is flushed as U+FFFD.
        let (emits, tail) = stream(&t, &[byte_id(b'c'), 2003]);
        assert_eq!(
            (emits.concat(), tail.as_str()),
            ("c".to_string(), "\u{FFFD}")
        );
        // An unknown id is an error and leaves the decoder as it was.
        let mut d = StreamDecoder::new();
        let mut out = String::new();
        d.push(&t, byte_id(0xE4), &mut out).unwrap();
        assert!(d.push(&t, 7, &mut out).is_err());
        d.push(&t, byte_id(0xB8), &mut out).unwrap();
        d.push(&t, byte_id(0xAD), &mut out).unwrap();
        assert_eq!(out, "中");
    }

    #[test]
    fn stream_decoder_equals_the_per_segment_lossy_decode() {
        let t = byte_tokenizer();
        let mut state = 0x5eed_0a11_u64;
        let mut next = |n: u64| {
            state = state
                .wrapping_mul(6_364_136_223_846_793_005)
                .wrapping_add(1_442_695_040_888_963_407);
            (state >> 33) % n
        };
        let mut checked_invalid = 0;
        for _ in 0..3000 {
            let len = 1 + next(24) as usize;
            let ids: Vec<u32> = (0..len)
                .map(|_| match next(10) {
                    0 => PAGE,
                    1 => special::IMAGE,
                    2 | 3 => MULTI[next(MULTI.len() as u64) as usize].0,
                    // Lead and continuation bytes, often enough to split and
                    // to break characters.
                    4 => byte_id(0xC0 + next(0x38) as u8),
                    5 | 6 => byte_id(0x80 + next(0x40) as u8),
                    _ => byte_id(b'a' + next(26) as u8),
                })
                .collect();
            let full = lossy_segments(&t, &ids);
            let (emits, tail) = stream(&t, &ids);
            assert_eq!(format!("{}{tail}", emits.concat()), full, "{ids:?}");
            // Nothing emitted early is ever taken back, and every prefix
            // flushes to its own lossy decode.
            let mut d = StreamDecoder::new();
            let mut so_far = String::new();
            for (i, &id) in ids.iter().enumerate() {
                d.push(&t, id, &mut so_far).unwrap();
                assert!(full.starts_with(&so_far), "{ids:?} at {i}");
                let mut flushed = so_far.clone();
                d.clone().flush(&mut flushed);
                assert_eq!(flushed, lossy_segments(&t, &ids[..=i]), "{ids:?} at {i}");
            }
            match t.decode(&ids) {
                Ok(strict) => assert_eq!(strict, full, "{ids:?}"),
                Err(_) => checked_invalid += 1,
            }
        }
        assert!(
            checked_invalid > 100,
            "too few invalid sequences: {checked_invalid}"
        );
    }

    /// The real v1 tokenizer splits rarer CJK characters across byte tokens;
    /// streaming its ids must still reproduce the text, `<PAGE>` text markers
    /// included, with no error at any prefix.
    #[test]
    fn stream_decoder_reassembles_real_cjk_splits() {
        let Some(t) = load_real() else {
            return;
        };
        let text = "<PAGE>\n第一页：饕餮盛宴\n魑魅魍魉，犇骉鱻羴。\n<PAGE>\n第二页：垚与赑屃 🦀";
        let ids = t.encode(text).unwrap();
        let (emits, tail) = stream(&t, &ids);
        assert_eq!(emits.concat(), text);
        assert_eq!(tail, "");
        let split_prefixes = (1..ids.len())
            .filter(|&n| t.decode(&ids[..n]).is_err())
            .count();
        assert!(split_prefixes > 0, "no character was split across tokens");
    }

    // ── Unlimited-OCR-2 tokenizer (U2-A4) ────────────────────────────────────

    /// The v2 `tokenizer.json`: `FOCR_UOCR2_TOKENIZER_JSON`, else the one in the
    /// `FOCR_UOCR2_SOURCE_DIR` checkpoint directory; neither set → skip. A file
    /// other than the pinned v2 tokenizer is a setup error, not a skip.
    fn load_real_uocr2() -> Option<Tokenizer> {
        use sha2::Digest as _;
        let path = std::env::var_os("FOCR_UOCR2_TOKENIZER_JSON")
            .map(std::path::PathBuf::from)
            .or_else(|| {
                std::env::var_os("FOCR_UOCR2_SOURCE_DIR")
                    .map(|dir| std::path::PathBuf::from(dir).join("tokenizer.json"))
            });
        let Some(path) = path else {
            eprintln!(
                "SKIP uocr2 tokenizer conformance: set FOCR_UOCR2_TOKENIZER_JSON \
                 (or FOCR_UOCR2_SOURCE_DIR) to the pinned v2 tokenizer.json"
            );
            return None;
        };
        let bytes =
            std::fs::read(&path).unwrap_or_else(|error| panic!("read {}: {error}", path.display()));
        assert_eq!(
            format!("{:x}", sha2::Sha256::digest(&bytes)),
            UNLIMITED_OCR_V2_TOKENIZER_SHA256,
            "{} is not the pinned Unlimited-OCR-2 tokenizer.json",
            path.display()
        );
        Some(Tokenizer::from_json_bytes(&bytes).expect("the pinned v2 tokenizer.json loads"))
    }

    use crate::native_engine::model_arch::{
        UNLIMITED_OCR_V2_TOKENIZER_SHA256, UNLIMITED_OCR_V2_TOKENS,
    };

    fn ids_of(record: &serde_json::Value) -> Vec<u32> {
        record["ids"]
            .as_array()
            .expect("ids array")
            .iter()
            .map(|id| u32::try_from(id.as_u64().expect("id")).expect("u32 id"))
            .collect()
    }

    /// **U2-A4 — the Unlimited-OCR-2 token-id-EXACT conformance gate.** The
    /// contract of [`baidu_token_id_conformance_gate`] over
    /// `expected_uocr2.json`: v1's corpus plus `corpus_uocr2.txt`, generated
    /// by `FOCR_BAIDU_MODEL=unlimited-ocr-2 scripts/gen_token_id_fixtures.py`
    /// with the v2 reference stack's `tokenizers==0.21.4`. Every record also
    /// pins its skip-special decode, which must keep the four non-special v2
    /// tokens. The golden's own pins are checked even without the tokenizer.
    #[test]
    fn uocr2_token_id_conformance_gate() {
        const EXPECTED: &str = include_str!(concat!(
            env!("CARGO_MANIFEST_DIR"),
            "/tests/fixtures/tokenizer_baidu/expected_uocr2.json"
        ));
        let started = std::time::Instant::now();
        let v: serde_json::Value = serde_json::from_str(EXPECTED).unwrap();
        let meta = &v["_meta"];
        assert_eq!(
            meta["tokenizer_json_sha256"],
            UNLIMITED_OCR_V2_TOKENIZER_SHA256
        );
        assert_eq!(meta["model_id"], "unlimited-ocr-2");
        assert_eq!(meta["tokenizers_version"], "0.21.4");
        assert_eq!(meta["max_token_id"], special::CHILD);
        let cases = v["fixtures"].as_array().expect("fixtures array");
        assert_eq!(Some(cases.len() as u64), meta["num_cases"].as_u64());
        // The v2 inference prompts are in the golden, and each starts with one
        // `<PAGE>` `<image>` pair per page.
        for pages in [1, 3, 20] {
            let prompt = format!("{}Multi page parsing.", "<PAGE><image>".repeat(pages));
            let record = cases
                .iter()
                .find(|record| record["text"] == prompt.as_str())
                .unwrap_or_else(|| panic!("the {pages}-page prompt is in the golden"));
            assert_eq!(
                ids_of(record)[..2 * pages],
                [special::PAGE, special::IMAGE].repeat(pages)[..]
            );
        }
        let Some(t) = load_real_uocr2() else {
            return;
        };
        let mismatches = fixture_mismatches(&t, cases);
        assert_eq!(
            mismatches, 0,
            "tok_id_mismatch_count must be 0 (got {mismatches})"
        );
        for (content, id) in meta["anchor_ids"].as_object().expect("anchors") {
            let id = u32::try_from(id.as_u64().expect("anchor id")).expect("u32 id");
            assert_eq!(t.encode(content).unwrap(), vec![id], "{content}");
        }
        for (content, id) in meta["non_special_added_ids"]
            .as_object()
            .expect("v2 tokens")
        {
            let id = u32::try_from(id.as_u64().expect("v2 id")).expect("u32 id");
            assert_eq!(t.decode_skip_special(&[id]).unwrap(), *content);
        }
        for &(id, content) in UNLIMITED_OCR_V2_TOKENS {
            assert_eq!(t.token_to_id(content), Some(id), "{content}");
            assert_eq!(t.id_to_token(id), Some(content));
        }
        assert_eq!(t.vocab_size(), 128_000);
        assert_eq!(t.id_to_token(special::CHILD), Some("<CHILD>"));
        assert_eq!(
            t.id_to_token(special::CHILD + 1),
            None,
            "128830 is the highest id"
        );
        assert!(t.decode(&[special::CHILD + 1]).is_err());
        println!(
            "{}",
            serde_json::json!({
                "schema_version": 1, "ts": started.elapsed().as_secs_f64(),
                "test": "uocr2_token_id_conformance_gate", "case": "expected_uocr2",
                "run_seq": 0, "event": "result", "result": "pass",
                "elapsed_us": started.elapsed().as_micros() as u64,
                "cases": cases.len(), "tok_id_mismatch_count": mismatches,
            })
        );
    }

    /// v1's tokenizer has no `<PAGE>`: the text spells ordinary tokens there,
    /// so a v1 sidecar can never pass for a v2 one on the required-token check.
    #[test]
    fn v1_tokenizer_has_no_page_token() {
        let Some(t) = load_real() else {
            return;
        };
        assert_eq!(t.token_to_id("<PAGE>"), None);
        assert!(!t.encode("<PAGE>").unwrap().contains(&special::PAGE));
        assert_eq!(t.id_to_token(special::PAGE), None);
    }

    /// SplitMix64: a deterministic, dependency-free stream for the fuzz tests.
    struct SplitMix64(u64);

    impl SplitMix64 {
        fn next(&mut self) -> u64 {
            self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
            let mut z = self.0;
            z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
            z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
            z ^ (z >> 31)
        }

        fn below(&mut self, n: usize) -> usize {
            (self.next() % n as u64) as usize
        }
    }

    /// The v2 added tokens and `<image>`, which the fuzz interleaves with text.
    const UOCR2_FUZZ_TOKENS: [(&str, u32); 5] = [
        ("<PAGE>", special::PAGE),
        ("<BLOCK>", special::BLOCK),
        ("<FORK>", special::FORK),
        ("<CHILD>", special::CHILD),
        ("<image>", special::IMAGE),
    ];

    /// Near misses that must stay text (alone; they may still combine with
    /// neighbouring pieces into a whole token, which the oracle counts).
    const UOCR2_NEAR_MISSES: [&str; 12] = [
        "<PAGE", "PAGE>", "<page>", "< PAGE>", "<PAGE >", "<PAG", "E>", "<BLOC", "K>", "<FOR",
        "<CHIL", "D>",
    ];

    /// Text characters. Of the real v2 tokenizer's 834 added tokens, only the
    /// [`UOCR2_FUZZ_TOKENS`] are spelled entirely from these characters and the
    /// pieces above, so they are the only added tokens a fuzz string can hold.
    const UOCR2_FUZZ_CHARS: [char; 17] = [
        'a', 'Z', '1', '9', ' ', '\n', '\t', '中', '文', 'é', '🦀', '<', '>', 'P', 'A', 'G', 'E',
    ];

    /// One to twelve pieces, each a whole token, a near miss, or a short run
    /// of text characters.
    fn uocr2_fuzz_string(rng: &mut SplitMix64) -> String {
        let mut text = String::new();
        for _ in 0..1 + rng.below(12) {
            match rng.below(3) {
                0 => text.push_str(UOCR2_FUZZ_TOKENS[rng.below(UOCR2_FUZZ_TOKENS.len())].0),
                1 => text.push_str(UOCR2_NEAR_MISSES[rng.below(UOCR2_NEAR_MISSES.len())]),
                _ => {
                    for _ in 0..1 + rng.below(6) {
                        text.push(UOCR2_FUZZ_CHARS[rng.below(UOCR2_FUZZ_CHARS.len())]);
                    }
                }
            }
        }
        text
    }

    /// The added-token properties on one string:
    /// - encode keeps every added token whole: each occurrence becomes exactly
    ///   its one id. The surfaces are `<` + letters + `>`, so two occurrences
    ///   can never overlap and `str::matches` counts them exactly;
    /// - decode inverts encode, and the skip-special decode drops `<image>` only;
    /// - splitting is lossless, and re-splitting any segment yields that segment.
    fn check_added_token_properties(t: &Tokenizer, text: &str) -> Result<(), String> {
        let ids = t.encode(text).map_err(|error| format!("encode: {error}"))?;
        let decoded = t.decode(&ids).map_err(|error| format!("decode: {error}"))?;
        if decoded != text {
            return Err(format!("decode(encode) is {decoded:?}"));
        }
        for (content, id) in UOCR2_FUZZ_TOKENS {
            let want = text.matches(content).count();
            let got = ids.iter().filter(|&&x| x == id).count();
            if got != want {
                return Err(format!(
                    "{content}: {got} ids for {want} occurrences in {ids:?}"
                ));
            }
        }
        let skipped = t
            .decode_skip_special(&ids)
            .map_err(|error| format!("skip: {error}"))?;
        if skipped != text.replace("<image>", "") {
            return Err(format!("skip-special decode is {skipped:?}"));
        }
        let mut joined = String::new();
        for segment in t.split_on_added(text) {
            match segment {
                Segment::Text(run) => {
                    if !matches!(t.split_on_added(run)[..], [Segment::Text(again)] if again == run)
                    {
                        return Err(format!("text segment {run:?} splits again"));
                    }
                    joined.push_str(run);
                }
                Segment::Added(id) => {
                    let content = t.id_to_token(id).ok_or("unknown added id")?;
                    if !matches!(t.split_on_added(content)[..], [Segment::Added(again)] if again == id)
                    {
                        return Err(format!("added token {content:?} does not split to itself"));
                    }
                    joined.push_str(content);
                }
            }
        }
        if joined != text {
            return Err(format!("segments rejoin to {joined:?}"));
        }
        Ok(())
    }

    fn run_added_token_fuzz(t: &Tokenizer, test: &str, seed: u64, iterations: usize) {
        let started = std::time::Instant::now();
        let mut rng = SplitMix64(seed);
        for iteration in 0..iterations {
            let text = uocr2_fuzz_string(&mut rng);
            if let Err(why) = check_added_token_properties(t, &text) {
                panic!("{test}: seed {seed:#x} iteration {iteration}: {text:?}: {why}");
            }
        }
        println!(
            "{}",
            serde_json::json!({
                "schema_version": 1, "ts": started.elapsed().as_secs_f64(), "test": test,
                "case": "added_token_fuzz", "run_seq": 0, "event": "result", "result": "pass",
                "elapsed_us": started.elapsed().as_micros() as u64, "seed": seed,
                "iterations": iterations,
            })
        );
    }

    /// Always on: the added-token properties over random interleavings, on a
    /// byte-level tokenizer carrying v2's added-token table shape.
    #[test]
    fn uocr2_added_tokens_stay_whole_under_fuzz() {
        let t = byte_tokenizer_with(&[
            (special::IMAGE, "<image>", true),
            (special::PAGE, "<PAGE>", false),
            (special::BLOCK, "<BLOCK>", false),
            (special::FORK, "<FORK>", false),
            (special::CHILD, "<CHILD>", false),
        ]);
        assert_eq!(t.encode("<PAGE>").unwrap(), vec![special::PAGE]);
        assert_eq!(t.decode_skip_special(&[special::PAGE]).unwrap(), "<PAGE>");
        run_added_token_fuzz(
            &t,
            "uocr2_added_tokens_stay_whole_under_fuzz",
            0x0A4_F022,
            2_000,
        );
    }

    /// Model-gated: the same properties on the real v2 tokenizer, whose BPE
    /// merges and pre-tokenizer now act on the text between the tokens.
    #[test]
    fn uocr2_real_tokenizer_keeps_added_tokens_whole_under_fuzz() {
        let Some(t) = load_real_uocr2() else {
            return;
        };
        run_added_token_fuzz(
            &t,
            "uocr2_real_tokenizer_keeps_added_tokens_whole_under_fuzz",
            0x0A4_BEEF,
            1_000,
        );
    }

    // ── SmolLM2 scheme (C6, bd-3jo6.3.6) ─────────────────────────────────────

    /// A tiny synthetic SmolLM2-style `tokenizer.json`: the SmolLM2
    /// `pre_tokenizer` declaration (which selects [`PretokScheme::SmolLm2`]),
    /// a vocab covering letters/digits/space symbols, and added tokens
    /// carrying the pinned `<image>`/`<end_of_utterance>` ids.
    fn tiny_smollm2_json() -> String {
        r#"{
          "version": "1.0",
          "added_tokens": [
            {"id": 49190, "content": "<image>", "special": true},
            {"id": 49279, "content": "<end_of_utterance>", "special": true}
          ],
          "normalizer": null,
          "pre_tokenizer": {
            "type": "Sequence",
            "pretokenizers": [
              {"type": "Digits", "individual_digits": true},
              {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true}
            ]
          },
          "post_processor": null,
          "model": {
            "type": "BPE",
            "vocab": {
              "a": 0, "b": 1, "c": 2, "1": 3, "2": 4,
              "Ġ": 5, "ab": 6, "12": 7, "'": 8, "s": 9, "'s": 10
            },
            "merges": [
              ["a", "b"],
              ["1", "2"],
              ["'", "s"]
            ]
          }
        }"#
        .to_string()
    }

    fn tk_smollm2() -> Tokenizer {
        Tokenizer::from_json_bytes(tiny_smollm2_json().as_bytes()).expect("tiny smollm2 loads")
    }

    #[test]
    fn smollm2_scheme_is_detected_from_pre_tokenizer() {
        let t = tk_smollm2();
        assert_eq!(t.scheme(), PretokScheme::SmolLm2);
        // The synthetic Baidu fixture (no pre_tokenizer) stays DeepSeek-V2.
        assert_eq!(tk().scheme(), PretokScheme::DeepSeekV2);
    }

    #[test]
    fn smollm2_scheme_ids() {
        let t = tk_smollm2();
        assert_eq!(t.bos_id(), special_smollm2::BOS); // <|im_start|> = 1
        assert_eq!(t.eos_id(), special_smollm2::END_OF_UTTERANCE); // 49279
        assert_eq!(t.pad_id(), special_smollm2::PAD); // <|im_end|> = 2
        assert_eq!(t.image_id(), special_smollm2::IMAGE); // 49190
        // The DeepSeek scheme keeps the Baidu ids.
        let b = tk();
        assert_eq!(b.bos_id(), special::BOS);
        assert_eq!(b.eos_id(), special::EOS);
        assert_eq!(b.image_id(), special::IMAGE);
    }

    #[test]
    fn smollm2_digits_encode_individually() {
        let t = tk_smollm2();
        // The Digits stage isolates each digit into its own piece, so the
        // "1"+"2"→"12" merge can never apply across the piece boundary.
        assert_eq!(t.encode("12").unwrap(), vec![3, 4]);
        // Under the DeepSeek scheme the same vocab WOULD merge "12" (digit
        // groups of ≤3 stay one piece) — proving the scheme dispatch is live.
        // (Also rename `<image>`: the DeepSeek pin table requires 128815.)
        let baidu_style = tiny_smollm2_json()
            .replace(
                r#""pre_tokenizer": {
            "type": "Sequence",
            "pretokenizers": [
              {"type": "Digits", "individual_digits": true},
              {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true}
            ]
          },"#,
                "",
            )
            .replace("<image>", "<img2>");
        assert!(
            !baidu_style.contains("pre_tokenizer"),
            "test fixture drifted: the pre_tokenizer block was not stripped"
        );
        let b = Tokenizer::from_json_bytes(baidu_style.as_bytes()).expect("loads");
        assert_eq!(b.scheme(), PretokScheme::DeepSeekV2);
        assert_eq!(b.encode("12").unwrap(), vec![7]);
    }

    #[test]
    fn smollm2_contraction_merges() {
        let t = tk_smollm2();
        // "ab's": alt 2 "ab", alt 1 "'s" → merge "'"+"s" applies inside the
        // "'s" piece.
        assert_eq!(t.encode("ab's").unwrap(), vec![6, 10]);
    }

    #[test]
    fn smollm2_added_tokens_split() {
        let t = tk_smollm2();
        assert_eq!(
            t.encode("ab<image>ab").unwrap(),
            vec![6, special_smollm2::IMAGE, 6]
        );
        assert_eq!(
            t.decode_skip_special(&t.encode("ab<image>ab").unwrap())
                .unwrap(),
            "abab"
        );
    }

    #[test]
    fn smollm2_wrong_image_pin_is_rejected() {
        let bad = tiny_smollm2_json().replace("\"id\": 49190", "\"id\": 999");
        let err = Tokenizer::from_json_bytes(bad.as_bytes()).unwrap_err();
        assert!(matches!(err, FocrError::FormatMismatch(_)), "got {err:?}");
    }

    #[test]
    fn unknown_pre_tokenizer_is_rejected() {
        // A Whitespace pre-tokenizer is neither supported shape → fail closed.
        let bad = tiny_smollm2_json().replace(
            r#"{"type": "Digits", "individual_digits": true}"#,
            r#"{"type": "Whitespace"}"#,
        );
        let err = Tokenizer::from_json_bytes(bad.as_bytes()).unwrap_err();
        assert!(matches!(err, FocrError::FormatMismatch(_)), "got {err:?}");
        // So is a non-Sequence declaration.
        let bad2 = tiny_smollm2_json().replace(
            r#"{
            "type": "Sequence",
            "pretokenizers": [
              {"type": "Digits", "individual_digits": true},
              {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true}
            ]
          }"#,
            r#"{"type": "ByteLevel", "add_prefix_space": true, "use_regex": true}"#,
        );
        let err2 = Tokenizer::from_json_bytes(bad2.as_bytes()).unwrap_err();
        assert!(matches!(err2, FocrError::FormatMismatch(_)), "got {err2:?}");
    }

    // ── real-vocab conformance (env-gated on the SmolVLM2 tokenizer.json) ────
    // FOCR_SMOLVLM2_TOKENIZER_JSON points at the pinned file, else
    // $FOCR_SMOLVLM2_DIR/tokenizer.json (the zoo layout); absent ⇒ skip (the
    // model-gated pattern, matching load_real / FOCR_GOT_TIKTOKEN).

    pub(super) fn load_real_smolvlm2() -> Option<Tokenizer> {
        let path = std::env::var("FOCR_SMOLVLM2_TOKENIZER_JSON")
            .ok()
            .or_else(|| {
                std::env::var("FOCR_SMOLVLM2_DIR")
                    .ok()
                    .map(|d| format!("{d}/tokenizer.json"))
            });
        let Some(path) = path else {
            eprintln!(
                "SKIP smolvlm2 tokenizer conformance: set FOCR_SMOLVLM2_TOKENIZER_JSON \
                 (or FOCR_SMOLVLM2_DIR) to the pinned tokenizer.json"
            );
            return None;
        };
        let path = Path::new(&path);
        if !path.is_file() {
            eprintln!(
                "SKIP smolvlm2 tokenizer conformance: {} absent",
                path.display()
            );
            return None;
        }
        Some(Tokenizer::from_file(path).expect("pinned smolvlm2 tokenizer.json must load"))
    }

    #[test]
    fn smolvlm2_real_vocab_anchors() {
        let Some(t) = load_real_smolvlm2() else {
            return;
        };
        assert_eq!(t.scheme(), PretokScheme::SmolLm2);
        assert_eq!(t.vocab_size(), 49152);
        assert_eq!(t.encode("<|endoftext|>").unwrap(), vec![0]);
        assert_eq!(t.encode("<|im_start|>").unwrap(), vec![1]);
        assert_eq!(t.encode("<|im_end|>").unwrap(), vec![2]);
        assert_eq!(
            t.encode("<global-img>").unwrap(),
            vec![special_smollm2::GLOBAL_IMG]
        );
        assert_eq!(
            t.encode("<fake_token_around_image>").unwrap(),
            vec![special_smollm2::FAKE_AROUND_IMAGE]
        );
        assert_eq!(t.encode("<image>").unwrap(), vec![special_smollm2::IMAGE]);
        assert_eq!(
            t.encode("<end_of_utterance>").unwrap(),
            vec![special_smollm2::END_OF_UTTERANCE]
        );
        // The 6×6 row-marker grid brackets (spec §5).
        assert_eq!(t.encode("<row_1_col_1>").unwrap(), vec![49153]);
        assert_eq!(t.encode("<row_6_col_6>").unwrap(), vec![49188]);
    }

    /// **L0a — the SmolVLM2 tokenizer token-id-EXACT conformance gate (C6,
    /// bd-3jo6.3.6, spec §13).** Parses the committed golden fixtures —
    /// generated by the reference HF `tokenizers` engine over the pinned
    /// SmolVLM2 `tokenizer.json` via
    /// `scripts/gen_smolvlm2_token_id_fixtures.py` — and asserts our encoder
    /// reproduces every id stream AND our decoder every decoded string
    /// exactly. No SmolVLM2 vision/decoder parity bead may close while this is
    /// red (AGENTS.md doctrine).
    #[test]
    fn smolvlm2_token_id_conformance_gate() {
        let Some(t) = load_real_smolvlm2() else {
            return;
        };
        const EXPECTED: &str = include_str!(concat!(
            env!("CARGO_MANIFEST_DIR"),
            "/tests/fixtures/tokenizer_smolvlm2/expected.json"
        ));
        let v: serde_json::Value = serde_json::from_str(EXPECTED).unwrap();
        assert_eq!(
            v["_meta"]["tokenizer_json_sha256"].as_str().unwrap(),
            "5ece781dc8d2b2f3e2f289ca0ae50b17cfc27dd27bfe7971bb8241e0b964331a",
            "fixture was generated against a different tokenizer.json pin"
        );
        let cases = v["fixtures"].as_array().expect("fixtures array");
        let num_cases = v["_meta"]["num_cases"].as_u64().unwrap() as usize;
        assert_eq!(cases.len(), num_cases, "fixture _meta.num_cases drift");
        assert!(
            cases.len() >= 100,
            "conformance corpus must stay >= 100 cases"
        );
        let mut mismatches = 0usize;
        for rec in cases {
            let text = rec["text"].as_str().unwrap();
            let want: Vec<u32> = rec["ids"]
                .as_array()
                .unwrap()
                .iter()
                .map(|x| x.as_u64().unwrap() as u32)
                .collect();
            let got = t.encode(text).unwrap();
            if got != want {
                let pos = got
                    .iter()
                    .zip(&want)
                    .position(|(a, b)| a != b)
                    .unwrap_or_else(|| got.len().min(want.len()));
                eprintln!(
                    "ENC MISMATCH {{\"case\": {text:?}, \"len\": {}, \"mismatch_pos\": {pos}}}\n  \
                     got  {got:?}\n  want {want:?}",
                    want.len()
                );
                mismatches += 1;
            }
            let want_decoded = rec["decoded"].as_str().unwrap();
            let got_decoded = t.decode(&want).unwrap();
            if got_decoded != want_decoded {
                eprintln!(
                    "DEC MISMATCH {{\"case\": {text:?}}}\n  \
                     got  {got_decoded:?}\n  want {want_decoded:?}"
                );
                mismatches += 1;
            }
        }
        assert_eq!(
            mismatches, 0,
            "tok_id_mismatch_count must be 0 (got {mismatches})"
        );
    }

    // ── OneChart / OPT scheme (D9, bd-3jo6.4.9) ──────────────────────────────

    #[test]
    fn opt_files_load_and_scheme_dispatch() {
        // Tiny synthetic triple: vocab covers letters + Ġ, merges drive the
        // BPE loop, added tokens carry the census pins.
        let vocab = r#"{"a":0,"b":1,"c":2,"1":3,"2":4,"Ġ":5,"ab":6,"12":7}"#.as_bytes();
        let merges = b"#version: 0.2\na b\n1 2\n";
        let added = br#"{"<imgpad>":50265,"<img>":50266,"</img>":50267,"<Number>":50268}"#;
        let t = Tokenizer::from_opt_files(vocab, merges, added).expect("opt files load");
        assert_eq!(t.scheme(), PretokScheme::Gpt2);
        // GPT-2 scheme has NO Digits stage: "12" stays one piece and merges.
        assert_eq!(t.encode("12").unwrap(), vec![7]);
        assert_eq!(t.encode("ab").unwrap(), vec![6]);
        // Added tokens split + resolve; skip_special drops them.
        assert_eq!(
            t.encode("ab<imgpad>ab").unwrap(),
            vec![6, special_opt::IMG_PAD, 6]
        );
        assert_eq!(
            t.decode_skip_special(&t.encode("ab<Number>").unwrap())
                .unwrap(),
            "ab"
        );
        // Scheme-aware ids (census §4).
        assert_eq!(t.bos_id(), special_opt::BOS_EOS);
        assert_eq!(t.eos_id(), special_opt::BOS_EOS);
        assert_eq!(t.pad_id(), special_opt::PAD);
        assert_eq!(t.image_id(), special_opt::IMG_PAD);
        // A wrong pin fails loud.
        let bad = br#"{"<imgpad>":999}"#;
        let err = Tokenizer::from_opt_files(vocab, merges, bad).unwrap_err();
        assert!(matches!(err, FocrError::FormatMismatch(_)), "got {err:?}");
        // Missing dir → ModelNotFound.
        let err = Tokenizer::from_opt_dir(Path::new("/nonexistent")).unwrap_err();
        assert!(matches!(err, FocrError::ModelNotFound(_)), "got {err:?}");
    }

    /// **D9 — the OneChart tokenizer token-id-EXACT conformance gate**
    /// (skip-with-SUCCESS without `FOCR_ONECHART_DIR`): our OPT loader +
    /// GPT-2 scheme must reproduce every id stream of the committed golden
    /// fixtures (generated by the slow `transformers` `GPT2Tokenizer` over
    /// the pinned `vocab.json`+`merges.txt`+`added_tokens.json`).
    #[test]
    fn onechart_token_id_conformance_gate() {
        let Ok(dir) = std::env::var("FOCR_ONECHART_DIR") else {
            eprintln!("SKIP onechart tokenizer conformance: set FOCR_ONECHART_DIR");
            return;
        };
        let t = Tokenizer::from_opt_dir(Path::new(&dir)).expect("onechart tokenizer loads");
        assert_eq!(t.vocab_size(), 50_265);
        const EXPECTED: &str = include_str!(concat!(
            env!("CARGO_MANIFEST_DIR"),
            "/tests/fixtures/tokenizer_onechart/expected.json"
        ));
        let v: serde_json::Value = serde_json::from_str(EXPECTED).unwrap();
        let cases = v["fixtures"].as_object().expect("fixtures object");
        assert!(cases.len() >= 25, "conformance corpus shrank");
        let mut mismatches = 0usize;
        for (text, want) in cases {
            let want: Vec<u32> = want
                .as_array()
                .unwrap()
                .iter()
                .map(|x| x.as_u64().unwrap() as u32)
                .collect();
            let got = t.encode(text).unwrap();
            if got != want {
                eprintln!("ENC MISMATCH {text:?}\n  got  {got:?}\n  want {want:?}");
                mismatches += 1;
            }
        }
        eprintln!("[D9] {} cases, {mismatches} mismatches", cases.len());
        assert_eq!(mismatches, 0, "tok_id_mismatch_count must be 0");
    }
}