splintr 0.11.0

//! Parsers for the shared sections of a HuggingFace `tokenizer.json`:
//! `pre_tokenizer`, `normalizer`, and `added_tokens`. These are backend-agnostic
//! — the family-specific builders in [`super`] consume their output.

use serde_json::Value;

use super::super::added::{AddedToken, AddedTokenSet};
use super::super::normalizer::NormOp;
use super::super::precompiled::Precompiled;
use super::super::tokenizer::{GPT2_PATTERN, SENTENCEPIECE_PATTERN};
use super::HfJsonError;

/// How input is split before the model runs, distilled to what splintr needs.
#[derive(Debug, Clone)]
pub(super) struct PreTokenization {
    /// Whether tokens are byte-level encoded (GPT-2/Whisper/Llama3 style).
    pub byte_level: bool,
    /// The pre-tokenization split regex.
    pub pattern: String,
    /// Prepend a space to the input (ByteLevel/Metaspace `add_prefix_space`, or
    /// Metaspace `prepend_scheme` != "never").
    pub add_prefix_space: bool,
    /// Whether a concrete splitter was recognized (ByteLevel/Metaspace/Split). If
    /// false, `pattern` is the GPT-2 default — only a sound choice when no
    /// pre-tokenizer was declared (see the caller's guess guard).
    pub anchored: bool,
    /// Pre-tokenizer `type`s present in the json that this distiller does not
    /// itself model (they may still be handled by the multi-stage engine).
    pub unknown: Vec<String>,
}

/// Walk a `pre_tokenizer` value (possibly a `Sequence`) and distill it to a
/// byte-level flag plus a split regex.
///
/// - `ByteLevel` anywhere ⇒ byte-level encoding; default to [`GPT2_PATTERN`]
///   unless an explicit `Split` regex is present.
/// - `Split { pattern: Regex|String }` ⇒ use that regex.
/// - `Metaspace` (SentencePiece-style) ⇒ [`SENTENCEPIECE_PATTERN`].
/// - Anything else / absent ⇒ non-byte-level, [`GPT2_PATTERN`] fallback.
pub(super) fn parse_pre_tokenizer(pre: Option<&Value>) -> PreTokenization {
    let mut byte_level = false;
    let mut split_regex: Option<String> = None;
    let mut metaspace = false;
    // None until a ByteLevel/Metaspace node sets it; defaulted at the end.
    let mut add_prefix_space: Option<bool> = None;
    // Pre-tokenizer types we neither parse nor handle implicitly via a backend.
    let mut unknown: Vec<String> = Vec::new();

    fn walk(
        v: &Value,
        byte_level: &mut bool,
        split_regex: &mut Option<String>,
        metaspace: &mut bool,
        add_prefix_space: &mut Option<bool>,
        unknown: &mut Vec<String>,
    ) {
        match v.get("type").and_then(Value::as_str) {
            Some("ByteLevel") => {
                *byte_level = true;
                if let Some(b) = v.get("add_prefix_space").and_then(Value::as_bool) {
                    *add_prefix_space = Some(b);
                }
            }
            Some("Metaspace") => {
                *metaspace = true;
                // Newer configs use `prepend_scheme` ("always"/"first"/"never");
                // older ones use `add_prefix_space`.
                if let Some(scheme) = v.get("prepend_scheme").and_then(Value::as_str) {
                    *add_prefix_space = Some(scheme != "never");
                } else if let Some(b) = v.get("add_prefix_space").and_then(Value::as_bool) {
                    *add_prefix_space = Some(b);
                }
            }
            Some("Split") if split_regex.is_none() => {
                // pattern is {"Regex": "..."} or {"String": "..."}.
                if let Some(re) = v.get("pattern").and_then(|p| {
                    p.get("Regex")
                        .and_then(Value::as_str)
                        .or_else(|| p.get("String").and_then(Value::as_str))
                }) {
                    *split_regex = Some(re.to_string());
                }
            }
            // Whitespace-only splitters are subsumed by both our SentencePiece
            // (whitespace-split) and byte-level paths, so they need no pattern.
            Some("Split") | Some("Whitespace") | Some("WhitespaceSplit") => {}
            Some("Sequence") => {
                if let Some(list) = v.get("pretokenizers").and_then(Value::as_array) {
                    for item in list {
                        walk(
                            item,
                            byte_level,
                            split_regex,
                            metaspace,
                            add_prefix_space,
                            unknown,
                        );
                    }
                }
            }
            Some(other) => unknown.push(other.to_string()),
            None => {}
        }
    }

    if let Some(pre) = pre {
        walk(
            pre,
            &mut byte_level,
            &mut split_regex,
            &mut metaspace,
            &mut add_prefix_space,
            &mut unknown,
        );
    }

    let pattern = match (&split_regex, metaspace) {
        (Some(re), _) => re.clone(),
        (None, true) => SENTENCEPIECE_PATTERN.to_string(),
        (None, false) => GPT2_PATTERN.to_string(),
    };

    PreTokenization {
        byte_level,
        pattern,
        // ByteLevel/Metaspace default `add_prefix_space` to true in HF when the
        // field is absent; real configs set it explicitly.
        add_prefix_space: add_prefix_space.unwrap_or(metaspace || byte_level),
        anchored: byte_level || metaspace || split_regex.is_some(),
        unknown,
    }
}

/// BERT-family normalizer flags consumed by the WordPiece backend.
#[derive(Debug, Clone)]
pub(super) struct BertNorm {
    pub lowercase: bool,
    /// Strip accents (`BertNormalizer.strip_accents`), already resolved from the
    /// json's tri-state — see [`parse_bert_norm`]. Independent of `lowercase`.
    pub strip_accents: bool,
    /// Isolate CJK ideographs (`handle_chinese_chars`); defaults to true.
    pub handle_chinese_chars: bool,
    /// Strip control/format chars and normalize whitespace (`clean_text`);
    /// defaults to true.
    pub clean_text: bool,
}

/// State threaded through the [`parse_bert_norm`] walk. A struct rather than a
/// pile of `&mut` arguments because `strip_accents` is only interpretable next
/// to the node that set it.
struct BertNormWalk {
    lowercase: bool,
    /// `None` until a node settles it; `Some` once a `BertNormalizer` (or an
    /// NFD-preceded `StripAccents`) has spoken. Resolved by the caller.
    strip_accents: Option<bool>,
    handle_chinese_chars: bool,
    clean_text: bool,
    /// Whether a decomposing normalizer (NFD/NFKD) has already run in this
    /// sequence — see the `StripAccents` arm for why that matters.
    decomposed: bool,
}

/// Extract the WordPiece-relevant flags from a (`BertNormalizer`-shaped)
/// normalizer. WordPiece's `BasicTokenizer` interleaves CJK splitting with
/// casing, so it consumes flags rather than the ordered op pipeline.
///
/// `strip_accents` is a **tri-state** in the json (`true` / `false` /
/// absent-or-`null`) and is NOT a synonym for `lowercase`. HuggingFace's
/// `BertNormalizer::normalize` computes `strip_accents.unwrap_or(lowercase)`,
/// so the absent form merely *defaults* to `lowercase` while an explicit value
/// wins on its own — cased multilingual BERT ships `strip_accents: false`
/// alongside `lowercase: false`, and a checkpoint whose vocabulary keeps
/// accented forms is mis-tokenized if the two are coupled.
///
/// Measured against `tokenizers` 0.22.1 on a WordPiece fixture holding both
/// `cafe` and `café` (ids 4 and 5), the three cases are:
///
/// | `BertNormalizer` | `"café"` |
/// |---|---|
/// | `lowercase: true,  strip_accents: null`  | `[4]` (`cafe`) |
/// | `lowercase: true,  strip_accents: false` | `[5]` (`café`) |
/// | `lowercase: false, strip_accents: true`  | `[4]` (`cafe`) |
///
/// The `Sequence` walk is deliberately asymmetric about the two sibling node
/// types, and both halves were measured on the same fixture:
///
/// - A standalone `Lowercase` node lowercases and says **nothing** about
///   accents: `Sequence[BertNormalizer{lowercase: false, strip_accents: null},
///   Lowercase]` yields `[5]` (`café` kept). So the `null` default resolves
///   against the `BertNormalizer`'s **own** `lowercase` field, not against
///   whatever else in the sequence happens to lowercase.
/// - A standalone `StripAccents` node does **not** imply BERT-style accent
///   stripping: HF's `StripAccents` only drops nonspacing marks and never
///   decomposes, so on ordinary (NFC) text it is a no-op — `Sequence[StripAccents]`
///   yields `[5]` (`café` kept). Honoring it as a flag would over-strip, because
///   this backend's stripper NFD-decomposes first (as BERT's own does). It is
///   therefore only honored when a decomposing `NFD`/`NFKD` node precedes it,
///   which is the one arrangement HF actually strips under
///   (`Sequence[NFD, StripAccents]` yields `[4]`).
pub(super) fn parse_bert_norm(norm: Option<&Value>) -> BertNorm {
    let mut state = BertNormWalk {
        lowercase: false,
        strip_accents: None,
        handle_chinese_chars: true,
        clean_text: true,
        decomposed: false,
    };

    fn walk(v: &Value, st: &mut BertNormWalk) {
        match v.get("type").and_then(Value::as_str) {
            Some("Lowercase") => st.lowercase = true,
            Some("NFD") | Some("NFKD") => st.decomposed = true,
            Some("StripAccents") if st.decomposed => st.strip_accents = Some(true),
            Some("BertNormalizer") => {
                let lc = v.get("lowercase").and_then(Value::as_bool).unwrap_or(false);
                if lc {
                    st.lowercase = true;
                }
                // Resolved here, against this node's own `lowercase`, because
                // `null` means "follow *my* lowercase" — not the sequence's.
                st.strip_accents = Some(
                    v.get("strip_accents")
                        .and_then(Value::as_bool)
                        .unwrap_or(lc),
                );
                st.handle_chinese_chars = v
                    .get("handle_chinese_chars")
                    .and_then(Value::as_bool)
                    .unwrap_or(true);
                st.clean_text = v.get("clean_text").and_then(Value::as_bool).unwrap_or(true);
            }
            Some("Sequence") => {
                if let Some(list) = v.get("normalizers").and_then(Value::as_array) {
                    for item in list {
                        walk(item, st);
                    }
                }
            }
            _ => {}
        }
    }
    if let Some(norm) = norm {
        walk(norm, &mut state);
    }
    BertNorm {
        lowercase: state.lowercase,
        // Nothing in the file claimed accents either way: no stripping. A bare
        // `Lowercase` normalizer (no `BertNormalizer`) lands here, and HF's
        // `Lowercase` leaves accents intact.
        strip_accents: state.strip_accents.unwrap_or(false),
        handle_chinese_chars: state.handle_chinese_chars,
        clean_text: state.clean_text,
    }
}

/// Parse a `normalizer` value into an ordered list of [`NormOp`]s, flattening
/// `Sequence`s in order.
///
/// A normalizer is an ordered pipeline with no implicit backend fallback, so an
/// unrecognized step (or a `Replace` regex that fails to compile) is a genuine
/// correctness gap: dropping it silently would mis-normalize and produce wrong
/// tokens with no signal. Such cases are surfaced as an error instead.
pub(super) fn parse_norm_ops(norm: Option<&Value>) -> Result<Vec<NormOp>, HfJsonError> {
    let mut ops = Vec::new();
    let mut unknown: Vec<String> = Vec::new();
    let mut bad_regex: Vec<String> = Vec::new();

    fn walk(
        v: &Value,
        ops: &mut Vec<NormOp>,
        unknown: &mut Vec<String>,
        bad_regex: &mut Vec<String>,
    ) {
        match v.get("type").and_then(Value::as_str) {
            Some("NFC") => ops.push(NormOp::Nfc),
            Some("NFD") => ops.push(NormOp::Nfd),
            Some("NFKC") => ops.push(NormOp::Nfkc),
            Some("NFKD") => ops.push(NormOp::Nfkd),
            Some("Lowercase") => ops.push(NormOp::Lowercase),
            Some("StripAccents") => ops.push(NormOp::StripAccents),
            Some("Nmt") => ops.push(NormOp::Nmt),
            Some("Prepend") => {
                if let Some(p) = v.get("prepend").and_then(Value::as_str) {
                    ops.push(NormOp::Prepend(p.to_string()));
                }
            }
            Some("Strip") => ops.push(NormOp::Strip {
                left: v.get("strip_left").and_then(Value::as_bool).unwrap_or(true),
                right: v
                    .get("strip_right")
                    .and_then(Value::as_bool)
                    .unwrap_or(true),
            }),
            Some("Replace") => {
                let content = v
                    .get("content")
                    .and_then(Value::as_str)
                    .unwrap_or("")
                    .to_string();
                if let Some(p) = v.get("pattern") {
                    if let Some(s) = p.get("String").and_then(Value::as_str) {
                        ops.push(NormOp::ReplaceStr {
                            from: s.to_string(),
                            to: content,
                        });
                    } else if let Some(re) = p.get("Regex").and_then(Value::as_str) {
                        match NormOp::replace_regex(re, content) {
                            Some(op) => ops.push(op),
                            None => bad_regex.push(re.to_string()),
                        }
                    }
                }
            }
            Some("Precompiled") => {
                if let Some(b64) = v.get("precompiled_charsmap").and_then(Value::as_str) {
                    use base64::Engine;
                    if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(b64) {
                        if let Some(pc) = Precompiled::from_bytes(&bytes) {
                            ops.push(NormOp::Precompiled(pc));
                        }
                    }
                }
            }
            // A `BertNormalizer` inside an SP/Unigram graph: expand to its ordered
            // effect (NFD + StripAccents when stripping, then Lowercase).
            Some("BertNormalizer") => {
                let lc = v.get("lowercase").and_then(Value::as_bool).unwrap_or(false);
                let strip = match v.get("strip_accents").and_then(Value::as_bool) {
                    Some(b) => b,
                    None => lc,
                };
                if strip {
                    ops.push(NormOp::Nfd);
                    ops.push(NormOp::StripAccents);
                }
                if lc {
                    ops.push(NormOp::Lowercase);
                }
            }
            Some("Sequence") => {
                if let Some(list) = v.get("normalizers").and_then(Value::as_array) {
                    for item in list {
                        walk(item, ops, unknown, bad_regex);
                    }
                }
            }
            Some(other) => unknown.push(other.to_string()),
            None => {}
        }
    }

    if let Some(norm) = norm {
        walk(norm, &mut ops, &mut unknown, &mut bad_regex);
    }
    if !unknown.is_empty() {
        return Err(HfJsonError::UnsupportedNormalizer(unknown.join(", ")));
    }
    if !bad_regex.is_empty() {
        return Err(HfJsonError::InvalidNormalizerRegex(bad_regex.join(", ")));
    }
    Ok(ops)
}

/// Collect **all** `added_tokens` into a content → [`AddedToken`] set.
///
/// HuggingFace matches every added token during encoding — both `special` ones
/// (`<|endoftext|>`) and non-special content tokens (e.g. gpt-neox's whitespace
/// runs, deepseek's byte chars) — so the matcher must know all of them, not just
/// the special-flagged ones.
///
/// Each entry's `lstrip`/`rstrip` booleans are read here rather than assumed
/// false: XLM-RoBERTa-family vocabularies (bge-m3, bge-reranker-v2-m3, and most
/// multilingual embedding models) declare `<mask>` with `lstrip: true` while
/// leaving it off on their four other added tokens, so the flags are only
/// correct when taken per token from the file. Both default to `false` when
/// absent, which is `tokenizers`' own default for an `AddedToken`.
pub(in crate::core) fn parse_special_tokens(root: &Value) -> AddedTokenSet {
    let mut specials = AddedTokenSet::new();
    if let Some(list) = root.get("added_tokens").and_then(Value::as_array) {
        for t in list {
            if let (Some(content), Some(id)) = (
                t.get("content").and_then(Value::as_str),
                t.get("id").and_then(Value::as_u64),
            ) {
                specials.insert(
                    content,
                    AddedToken {
                        id: id as u32,
                        lstrip: t.get("lstrip").and_then(Value::as_bool).unwrap_or(false),
                        rstrip: t.get("rstrip").and_then(Value::as_bool).unwrap_or(false),
                    },
                );
            }
        }
    }
    specials
}

/// Ids of `added_tokens` flagged `special: true` — dropped on decode to match
/// HuggingFace's default `skip_special_tokens=true`. Non-special added tokens
/// (e.g. gpt-neox whitespace runs) are kept.
pub(super) fn parse_special_decode_ids(root: &Value) -> rustc_hash::FxHashSet<u32> {
    let mut ids = rustc_hash::FxHashSet::default();
    if let Some(list) = root.get("added_tokens").and_then(Value::as_array) {
        for t in list {
            if t.get("special").and_then(Value::as_bool).unwrap_or(true) {
                if let Some(id) = t.get("id").and_then(Value::as_u64) {
                    ids.insert(id as u32);
                }
            }
        }
    }
    ids
}

/// Find the id of the first matching token content in `added_tokens`.
pub(in crate::core) fn find_added_token(root: &Value, candidates: &[&str]) -> Option<u32> {
    let list = root.get("added_tokens").and_then(Value::as_array)?;
    for cand in candidates {
        for t in list {
            if t.get("content").and_then(Value::as_str) == Some(cand) {
                return t.get("id").and_then(Value::as_u64).map(|n| n as u32);
            }
        }
    }
    None
}