gigastt-core 2.17.0

//! Contextual hotword biasing for the greedy RNN-T decode loop.
//!
//! Shallow-fusion biasing steers the greedy transducer toward a curated set of
//! phrases (brands, names, domain terms) without a beam search. Each hotword is
//! tokenized to the id sequence the active head would emit (via
//! [`Tokenizer::encode_phrase`](super::tokenizer::Tokenizer::encode_phrase), so
//! it adapts to whichever vocab is loaded) and stored in a small prefix trie.
//!
//! During decode, a [`BiasState`] tracks which hotword prefixes are currently
//! "active" given the recently emitted tokens. Before the argmax over the
//! joiner logits, [`Biaser::boost_logits`] adds a fixed boost to the logits of
//! the token-ids that would extend an active prefix. A token that completes /
//! advances a prefix advances the state; anything else resets it (while still
//! letting a fresh hotword start). Blank frames leave the prefix state
//! unchanged — they emit no label, so a partially-matched hotword survives the
//! gaps between its tokens.
//!
//! The [`Biaser`] itself is immutable after construction and shared across the
//! session pool via `&Biaser`; the only mutable per-decode bookkeeping lives in
//! [`BiasState`], created fresh for each decode. When no hotwords are
//! configured the engine holds no biaser at all and the decode path is
//! byte-for-byte unchanged.

use super::tokenizer::Tokenizer;

/// Encode `phrase` in whatever spelling the active vocab can actually
/// represent, or `None` if none of them fit.
///
/// The heads disagree about spelling: the `e2e_rnnt` BPE vocab carries case,
/// the `rnnt` char vocab is 32 lowercase Cyrillic letters with no `ё`, and the
/// multilingual one adds Latin and `ё`. Users write a glossary the way the
/// words are written — `Гигаэм`, `AmoCRM`, `Пётр` — so the phrase is tried as
/// spelled first (the only form a cased vocab wants), then lowercased, then
/// with `ё` folded to the `е` a head without `ё` emits in its place. The first
/// spelling that encodes whole wins; nothing beyond that is guessed, so a Latin
/// brand still cannot be biased on a Cyrillic-only head.
fn encode_representable(tokenizer: &Tokenizer, phrase: &str) -> Option<Vec<usize>> {
    if let Some(ids) = tokenizer.encode_phrase(phrase) {
        return Some(ids);
    }
    let lowercased = phrase.to_lowercase();
    if lowercased != phrase
        && let Some(ids) = tokenizer.encode_phrase(&lowercased)
    {
        return Some(ids);
    }
    let folded = lowercased.replace('ё', "е");
    if folded != lowercased {
        return tokenizer.encode_phrase(&folded);
    }
    None
}

/// One node of the hotword prefix trie. The root is index 0.
struct TrieNode {
    /// Edges keyed by token id → child node index.
    children: std::collections::HashMap<usize, usize>,
    /// True when a hotword phrase ends here. Only the beam search reads it: a
    /// finished phrase keeps the boost it was granted, an abandoned one does
    /// not.
    is_end: bool,
    /// Scored length of the shortest phrase running through this node, used to
    /// spread one phrase's worth of boost across its tokens.
    shortest_phrase: usize,
    /// Distance from the root; 1 for a phrase's first token.
    depth: usize,
    /// True when this node is only the word-boundary marker that opens a
    /// phrase. Matched, never paid for — it is the same token for every hotword
    /// ever configured.
    is_entry: bool,
    /// Boost paid for entering this node under the beam search's per-phrase
    /// budget. Zero at the entry marker.
    grant: f32,
}

impl TrieNode {
    fn new(depth: usize) -> Self {
        Self {
            children: std::collections::HashMap::new(),
            is_end: false,
            shortest_phrase: usize::MAX,
            depth,
            is_entry: false,
            grant: 0.0,
        }
    }
}

/// Where one beam sits in the hotword trie, and how much boost it has been
/// granted for a phrase it has not finished.
///
/// The greedy transducer path cannot take a bonus back — it has already emitted
/// the token — so it lives with a rationed boost instead
/// ([`super::decode::greedy_decode`]). A beam search can: a hypothesis that
/// walked halfway into a hotword and then left is refunded, so a partial match
/// wins nothing and only a phrase actually spoken keeps its advantage.
#[derive(Clone, Copy, Default)]
pub(crate) struct BiasPath {
    /// Current trie node; 0 is the root.
    node: usize,
    /// Boost granted so far for the unfinished phrase under way.
    pending: f32,
}

impl BiasPath {
    /// Boost this path has been granted for a phrase it has not finished.
    ///
    /// Owed back: a hypothesis still mid-phrase when the audio runs out never
    /// earned it, so the final ranking has to discount it. During the search
    /// the amount stays credited — that is what keeps a half-matched phrase in
    /// the beam long enough to finish.
    pub(crate) fn pending(&self) -> f32 {
        self.pending
    }
}

/// Compiled hotword biaser: a prefix trie over hotword token-id sequences plus
/// the additive logit boost. Immutable and shareable across inference sessions.
pub struct Biaser {
    nodes: Vec<TrieNode>,
    /// Additive boost applied to a continuation token's logit.
    boost: f32,
    /// Number of distinct hotword phrases successfully compiled.
    phrase_count: usize,
}

impl Biaser {
    /// Build a biaser from hotword token-id sequences and a boost. Sequences
    /// must be non-empty; empty ones are skipped. Returns `None` if no sequence
    /// survives (so callers treat "no usable hotwords" as biasing-off).
    ///
    /// Test-only, and the raw sequences carry no word-boundary marker: this is
    /// how the decode-loop tests build a biaser without a tokenizer, so nothing
    /// here is treated as a phrase-entry precondition.
    #[cfg(test)]
    pub(crate) fn from_sequences(sequences: Vec<Vec<usize>>, boost: f32) -> Option<Self> {
        Self::build(sequences, boost, false)
    }

    /// Compile `sequences` into the trie.
    ///
    /// `leading_is_entry` says the first token of every sequence is the
    /// word-boundary marker that [`Tokenizer::encode_phrase`] prepends. That
    /// token is a *precondition* for the phrase, not part of what makes it
    /// distinctive — it is the same token for every hotword ever configured, so
    /// paying for it means paying at every word boundary in the audio no matter
    /// what the glossary says. It is matched but never scored.
    fn build(sequences: Vec<Vec<usize>>, boost: f32, leading_is_entry: bool) -> Option<Self> {
        let mut nodes = vec![TrieNode::new(0)];
        let mut phrase_count = 0;
        let entry_tokens = usize::from(leading_is_entry);
        for seq in sequences {
            if seq.is_empty() {
                continue;
            }
            phrase_count += 1;
            // Tokens the boost is actually spread over.
            let scored = seq.len().saturating_sub(entry_tokens).max(1);
            let mut node = 0usize;
            for tok in seq {
                node = match nodes[node].children.get(&tok) {
                    Some(&child) => child,
                    None => {
                        let depth = nodes[node].depth + 1;
                        let child = nodes.len();
                        nodes.push(TrieNode::new(depth));
                        nodes[node].children.insert(tok, child);
                        child
                    }
                };
                nodes[node].shortest_phrase = nodes[node].shortest_phrase.min(scored);
            }
            nodes[node].is_end = true;
        }
        if phrase_count == 0 {
            return None;
        }
        for node in nodes.iter_mut().skip(1) {
            node.is_entry = leading_is_entry && node.depth == 1;
            node.grant = if node.is_entry {
                0.0
            } else {
                boost / node.shortest_phrase.max(1) as f32
            };
        }
        Some(Self {
            nodes,
            boost,
            phrase_count,
        })
    }

    /// Build a biaser from `(phrase, weight)` pairs, tokenizing each phrase with
    /// the active [`Tokenizer`]. `weight` scales the base `boost` per phrase
    /// (use `1.0` for the default). Phrases the tokenizer can't represent are
    /// dropped. Returns `None` if no phrase compiles or `boost <= 0`.
    pub fn from_phrases(
        tokenizer: &Tokenizer,
        phrases: &[(String, f32)],
        boost: f32,
    ) -> Option<Self> {
        if boost <= 0.0 {
            return None;
        }
        // Per-phrase weights are folded into the boost by storing the *highest*
        // requested boost on each trie edge would complicate the immutable
        // node layout; instead we keep a single base boost and treat the weight
        // as a phrase-level filter (weight <= 0 drops the phrase). A future
        // per-edge weight can extend TrieNode without touching the decode loop.
        let mut sequences = Vec::new();
        let mut dropped: Vec<&str> = Vec::new();
        for (phrase, weight) in phrases {
            if *weight <= 0.0 {
                continue;
            }
            match encode_representable(tokenizer, phrase) {
                Some(ids) => sequences.push(ids),
                None => dropped.push(phrase),
            }
        }
        if !dropped.is_empty() {
            // Named, not just counted: which phrases fell out is the whole
            // actionable content — a Cyrillic-only head can never represent a
            // Latin brand, and the only way a user learns that is by reading
            // its name here.
            tracing::warn!(
                "{} hotword phrase(s) dropped, not representable in the active vocab: {}",
                dropped.len(),
                dropped.join(", ")
            );
        }
        Self::build(sequences, boost, true)
    }

    /// Number of hotword phrases compiled into the trie.
    pub fn phrase_count(&self) -> usize {
        self.phrase_count
    }

    /// Score emitting `tok` from `path`, returning the log-domain delta to add
    /// to the hypothesis and the path that follows.
    ///
    /// Three cases, and the middle one is why this exists:
    /// - the token continues the phrase under way — grant the boost;
    /// - it does not, but starts some phrase — refund everything the abandoned
    ///   partial match was granted, then grant the boost for the new start;
    /// - it is not a hotword token at all — refund and return to the root.
    ///
    /// A node that ends a phrase clears the pending amount: the phrase was
    /// spoken, so its boost is earned and is never taken back.
    pub(crate) fn score_token(&self, path: BiasPath, tok: usize) -> (f32, BiasPath) {
        // A phrase is worth `boost` however long it is, so each of its tokens
        // is granted a share rather than the whole amount.
        //
        // A character vocabulary makes the difference stark: paid per token, a
        // nine-letter phrase would earn nine times the boost and outrank
        // whatever was actually said — `любовницы` really did displace
        // `люк кейдж` that way. Clawing the excess back on completion is worse
        // still: the correction lands as one large negative step and the
        // hypothesis that just finished the phrase gets pruned for it. Granting
        // the right amount from the start keeps every step small.
        let enter = |from: BiasPath, child: usize| {
            let share = self.nodes[child].grant;
            (
                share,
                BiasPath {
                    node: child,
                    // A finished phrase owes nothing back; an unfinished one
                    // owes everything it has been granted so far.
                    pending: if self.nodes[child].is_end {
                        0.0
                    } else {
                        from.pending + share
                    },
                },
            )
        };

        if let Some(&child) = self.nodes[path.node].children.get(&tok) {
            return enter(path, child);
        }

        // The phrase under way dies here: take back what it was granted, and
        // let a fresh phrase start on the same token.
        let refund = -path.pending;
        match self.nodes[0].children.get(&tok) {
            Some(&child) => {
                let (delta, next) = enter(BiasPath::default(), child);
                (refund + delta, next)
            }
            None => (refund, BiasPath::default()),
        }
    }

    /// Token ids that would extend the phrase `path` is in, plus every phrase
    /// start. A beam search hands these to the candidate set so a boosted
    /// continuation can be considered even when the acoustic model ranks it
    /// below the pruning cut — which is the entire point of biasing.
    pub(crate) fn continuations(&self, path: BiasPath, out: &mut Vec<usize>) {
        out.extend(self.nodes[path.node].children.keys().copied());
        if path.node != 0 {
            out.extend(self.nodes[0].children.keys().copied());
        }
    }

    /// Create a fresh per-decode prefix-tracking state rooted at the trie root.
    pub(crate) fn new_state(&self) -> BiasState {
        BiasState {
            // The root is always active so a new hotword can start at any token.
            active: vec![0],
        }
    }

    /// Add the boost to `logits` for every token id that extends a currently
    /// active hotword prefix. No-op when no active node has children (i.e. no
    /// hotword could continue here), so non-hotword regions are untouched.
    pub(crate) fn boost_logits(&self, state: &BiasState, logits: &mut [f32]) {
        for &node in &state.active {
            for (&tok, &child) in &self.nodes[node].children {
                if tok < logits.len() && !self.nodes[child].is_entry {
                    // The full boost, not the beam's per-phrase share: a greedy
                    // argmax decides each step on its own, so the step delta is
                    // the whole mechanism. Splitting it across a phrase's
                    // characters — right when totals compete in a beam — leaves
                    // a long hotword too weak to win any single step, which is
                    // to say it turns biasing off.
                    logits[tok] += self.boost;
                }
            }
        }
    }

    /// Advance the prefix state after a non-blank token `tok` was emitted.
    ///
    /// New active set = the children reached by `tok` from any previously active
    /// node, plus the root (so a fresh hotword can begin on the next token).
    /// Deduplicated to keep the active set small.
    pub(crate) fn advance(&self, state: &mut BiasState, tok: usize) {
        let mut next = Vec::new();
        for &node in &state.active {
            if let Some(&child) = self.nodes[node].children.get(&tok)
                && !next.contains(&child)
            {
                next.push(child);
            }
        }
        // The root stays active so biasing can restart at the next token.
        if !next.contains(&0) {
            next.push(0);
        }
        state.active = next;
    }
}

/// Per-decode hotword prefix-tracking state. Holds the set of trie nodes whose
/// prefix has been matched by the recently emitted tokens. Cheap to create;
/// one per [`greedy_decode`](super::decode::greedy_decode) call.
pub(crate) struct BiasState {
    active: Vec<usize>,
}

#[cfg(test)]
mod tests {
    use super::*;

    fn biaser(seqs: Vec<Vec<usize>>, boost: f32) -> Biaser {
        Biaser::from_sequences(seqs, boost).expect("non-empty sequences")
    }

    #[test]
    fn test_from_sequences_empty_returns_none() {
        assert!(Biaser::from_sequences(vec![], 5.0).is_none());
        assert!(Biaser::from_sequences(vec![vec![]], 5.0).is_none());
    }

    #[test]
    fn test_boost_applies_to_first_token_of_each_hotword() {
        // Two hotwords: [1,2] and [3]. At the root both 1 and 3 are boostable.
        // The greedy path pays the configured boost per step, whatever the
        // phrase's length — see `boost_logits`.
        let b = biaser(vec![vec![1, 2], vec![3]], 5.0);
        let state = b.new_state();
        let mut logits = vec![0.0; 5];
        b.boost_logits(&state, &mut logits);
        assert_eq!(logits[1], 5.0);
        assert_eq!(logits[3], 5.0);
        assert_eq!(logits[2], 0.0, "mid-hotword token not boosted at root");
        assert_eq!(logits[0], 0.0);
    }

    #[test]
    fn test_advance_then_boost_continuation() {
        // After emitting token 1, the hotword [1,2] should boost token 2.
        let b = biaser(vec![vec![1, 2]], 5.0);
        let mut state = b.new_state();
        b.advance(&mut state, 1);
        let mut logits = vec![0.0; 5];
        b.boost_logits(&state, &mut logits);
        assert_eq!(logits[2], 5.0, "continuation token boosted after prefix");
        // Token 1 is also boostable again because the root stays active.
        assert_eq!(logits[1], 5.0, "root keeps a fresh hotword start available");
    }

    #[test]
    fn test_advance_off_prefix_resets_to_root_only() {
        // Emit a non-matching token: only the root-level starts remain boosted.
        let b = biaser(vec![vec![1, 2]], 5.0);
        let mut state = b.new_state();
        b.advance(&mut state, 1); // on prefix [1]
        b.advance(&mut state, 9); // off prefix → reset to root
        let mut logits = vec![0.0; 5];
        b.boost_logits(&state, &mut logits);
        assert_eq!(logits[2], 0.0, "continuation no longer boosted after reset");
        assert_eq!(logits[1], 5.0, "root start still boosted");
    }

    #[test]
    fn test_shared_prefix_keeps_both_branches_active() {
        // Hotwords [1,2] and [1,3] share the first token.
        let b = biaser(vec![vec![1, 2], vec![1, 3]], 4.0);
        let mut state = b.new_state();
        b.advance(&mut state, 1);
        let mut logits = vec![0.0; 5];
        b.boost_logits(&state, &mut logits);
        assert_eq!(logits[2], 4.0);
        assert_eq!(logits[3], 4.0);
    }

    #[test]
    fn test_boost_ignores_out_of_range_token_id() {
        // A hotword token id beyond the logits length must not panic.
        let b = biaser(vec![vec![99]], 5.0);
        let state = b.new_state();
        let mut logits = vec![0.0; 5];
        b.boost_logits(&state, &mut logits); // no panic
        assert!(logits.iter().all(|&l| l == 0.0));
    }

    use crate::inference::tokenizer::Tokenizer;

    /// A char-vocab tokenizer covering the Cyrillic letters used below plus a
    /// `▁` word-boundary marker, so `encode_phrase` produces deterministic ids.
    fn char_tokenizer() -> Tokenizer {
        let tokens = vec![
            "а".to_string(),
            "б".to_string(),
            "в".to_string(),
            "г".to_string(),
            "д".to_string(),
            "\u{2581}".to_string(), // word-boundary marker
            "<unk>".to_string(),
            "<blk>".to_string(),
        ];
        Tokenizer::from_tokens(tokens)
    }

    #[test]
    fn the_phrase_entry_marker_is_never_boosted() {
        // Every phrase `encode_phrase` produces starts with the same
        // word-boundary marker, so paying for it means paying at every word
        // boundary in the audio whatever the glossary says. That is exactly
        // what happened: three different glossaries — one of them a word absent
        // from the recording — produced byte-identical transcripts, because the
        // only token being boosted was the space.
        let tok = char_tokenizer();
        let marker = tok.encode_phrase("а").expect("representable")[0];
        for phrases in [
            vec![("а".to_string(), 1.0)],
            vec![("аб".to_string(), 1.0)],
            vec![("вг".to_string(), 1.0), ("д".to_string(), 1.0)],
        ] {
            let b = Biaser::from_phrases(&tok, &phrases, 6.0).expect("compiles");
            let state = b.new_state();
            let mut logits = vec![0.0; 8];
            b.boost_logits(&state, &mut logits);
            assert_eq!(
                logits[marker], 0.0,
                "the entry marker took a boost for {phrases:?}"
            );
            assert!(
                logits.iter().all(|&l| l == 0.0),
                "nothing is boostable before a word boundary is emitted"
            );
        }
    }

    #[test]
    fn a_phrase_earns_one_boost_however_long_it_is() {
        // Paid per token, a nine-letter phrase would be worth nine boosts and
        // outrank whatever was actually said — that is how `любовницы`
        // displaced `люк кейдж` on real audio. Length must not buy rank.
        for len in [1usize, 3, 9] {
            let seq: Vec<usize> = (1..=len).collect();
            let b = Biaser::from_sequences(vec![seq.clone()], 6.0).expect("biaser");
            let mut path = BiasPath::default();
            let mut earned = 0.0;
            for &tok in &seq {
                let (delta, next) = b.score_token(path, tok);
                earned += delta;
                path = next;
            }
            assert!(
                (earned - 6.0).abs() < 1e-4,
                "a {len}-token phrase earned {earned}, expected the boost once"
            );
            assert_eq!(path.pending(), 0.0, "a finished phrase owes nothing back");
        }
    }

    #[test]
    fn an_abandoned_phrase_earns_nothing() {
        // Credit accrues while a phrase is under way so the beam keeps it
        // alive; walking away has to give all of it back, or a partial match
        // would be rewarded for going nowhere.
        let b = Biaser::from_sequences(vec![vec![1, 2, 3, 4]], 6.0).expect("biaser");
        let mut path = BiasPath::default();
        let mut earned = 0.0;
        for tok in [1usize, 2] {
            let (delta, next) = b.score_token(path, tok);
            earned += delta;
            path = next;
        }
        assert!(path.pending() > 0.0, "mid-phrase credit is outstanding");
        let (delta, path) = b.score_token(path, 99);
        earned += delta;
        assert!(earned.abs() < 1e-4, "abandoned phrase netted {earned}");
        assert_eq!(path.pending(), 0.0);
    }

    #[test]
    fn test_from_phrases_zero_boost_returns_none() {
        let tok = char_tokenizer();
        let phrases = vec![("аб".to_string(), 1.0)];
        assert!(Biaser::from_phrases(&tok, &phrases, 0.0).is_none());
    }

    #[test]
    fn test_from_phrases_negative_boost_returns_none() {
        let tok = char_tokenizer();
        let phrases = vec![("аб".to_string(), 1.0)];
        assert!(Biaser::from_phrases(&tok, &phrases, -3.0).is_none());
    }

    #[test]
    fn test_from_phrases_empty_slice_returns_none() {
        let tok = char_tokenizer();
        assert!(Biaser::from_phrases(&tok, &[], 5.0).is_none());
    }

    #[test]
    fn test_from_phrases_all_zero_weight_returns_none() {
        // Positive boost but every phrase filtered out by weight <= 0 → None.
        let tok = char_tokenizer();
        let phrases = vec![("аб".to_string(), 0.0), ("вг".to_string(), -1.0)];
        assert!(Biaser::from_phrases(&tok, &phrases, 5.0).is_none());
    }

    #[test]
    fn test_from_phrases_unrepresentable_only_returns_none() {
        // A phrase with no codepoints in the vocab is dropped; nothing survives.
        let tok = char_tokenizer();
        let phrases = vec![("xyz".to_string(), 1.0)];
        assert!(Biaser::from_phrases(&tok, &phrases, 5.0).is_none());
    }

    #[test]
    fn test_from_phrases_single_token_phrase_boosts_first_token() {
        // "а" encodes to a leading ▁ (id 5) then char id 0. The marker only
        // opens the phrase; the character is what gets paid for.
        let tok = char_tokenizer();
        let phrases = vec![("а".to_string(), 1.0)];
        let b = Biaser::from_phrases(&tok, &phrases, 7.0).expect("phrase compiles");
        assert_eq!(b.phrase_count(), 1);

        let ids = tok.encode_phrase("а").expect("representable");
        let mut state = b.new_state();
        let mut logits = vec![0.0; 8];
        b.boost_logits(&state, &mut logits);
        assert_eq!(logits[ids[0]], 0.0, "the boundary marker is never boosted");

        b.advance(&mut state, ids[0]);
        let mut logits = vec![0.0; 8];
        b.boost_logits(&state, &mut logits);
        assert_eq!(
            logits[ids[1]], 7.0,
            "a one-character phrase is worth it all"
        );
    }

    #[test]
    fn test_from_phrases_multi_token_phrase_boosts_continuation() {
        // "аб" → [▁(5), а(0), б(1)]. After advancing through the encoded
        // prefix, the next continuation token must be boosted.
        let tok = char_tokenizer();
        let phrases = vec![("аб".to_string(), 1.0)];
        let b = Biaser::from_phrases(&tok, &phrases, 4.0).expect("phrase compiles");
        assert_eq!(b.phrase_count(), 1);

        let ids = tok.encode_phrase("аб").expect("representable");
        assert_eq!(ids, vec![5, 0, 1]);

        let mut state = b.new_state();
        b.advance(&mut state, ids[0]); // ▁
        b.advance(&mut state, ids[1]); // а
        let mut logits = vec![0.0; 8];
        b.boost_logits(&state, &mut logits);
        assert_eq!(
            logits[ids[2]], 4.0,
            "third token boosted after two-token prefix"
        );
    }

    #[test]
    fn test_from_phrases_drops_unrepresentable_keeps_representable() {
        // One good phrase, one with an out-of-vocab codepoint. The good one
        // survives; the count reflects only the compiled phrase.
        let tok = char_tokenizer();
        let phrases = vec![("аб".to_string(), 1.0), ("аz".to_string(), 1.0)];
        let b = Biaser::from_phrases(&tok, &phrases, 5.0).expect("one phrase compiles");
        assert_eq!(b.phrase_count(), 1);
    }

    /// A lowercase Cyrillic char vocab without `ё`, the shape of the `rnnt`
    /// head's 34-token vocabulary.
    fn cyrillic_tokenizer() -> Tokenizer {
        let tokens = [
            "г", "и", "а", "э", "м", "п", "т", "р", "е", "\u{2581}", "<unk>", "<blk>",
        ];
        Tokenizer::from_tokens(tokens.iter().map(|t| (*t).to_string()).collect())
    }

    #[test]
    fn test_from_phrases_capitalized_phrase_is_kept() {
        // Users write brands capitalized; every shipped vocab is lowercase, so
        // the written form was dropped outright and glossaries silently lost
        // most of their entries.
        let tok = cyrillic_tokenizer();
        let phrases = vec![("Гигаэм".to_string(), 1.0)];
        let b = Biaser::from_phrases(&tok, &phrases, 5.0).expect("capitalized phrase compiles");
        assert_eq!(b.phrase_count(), 1);
        // It compiles to exactly the lowercase spelling's tokens.
        let ids = tok.encode_phrase("гигаэм").expect("representable");
        let mut state = b.new_state();
        b.advance(&mut state, ids[0]); // past the boundary marker
        let mut logits = vec![0.0; 12];
        b.boost_logits(&state, &mut logits);
        assert!(
            logits[ids[1]] > 0.0,
            "the lowercased spelling is what biases"
        );
    }

    #[test]
    fn test_from_phrases_yo_folds_to_e_when_the_vocab_lacks_it() {
        // A head with no `ё` token emits `е` in its place, so that is the
        // spelling a hotword has to match.
        let tok = cyrillic_tokenizer();
        assert!(tok.encode_phrase("пётр").is_none(), "vocab has no ё");
        let phrases = vec![("Пётр".to_string(), 1.0)];
        let b = Biaser::from_phrases(&tok, &phrases, 5.0).expect("ё folds to е");
        assert_eq!(b.phrase_count(), 1);
    }

    #[test]
    fn test_from_phrases_cased_vocab_keeps_the_written_spelling() {
        // The `e2e_rnnt` BPE vocab carries case. Lowercasing unconditionally
        // would re-tokenize — and worsen — phrases that already fit.
        let tokens = ["Аб", "а", "б", "\u{2581}", "<unk>", "<blk>"];
        let tok = Tokenizer::from_tokens(tokens.iter().map(|t| (*t).to_string()).collect());
        let written = tok.encode_phrase("Аб").expect("cased vocab represents it");
        let b = Biaser::from_phrases(&tok, &[("Аб".to_string(), 1.0)], 5.0).expect("compiles");
        let mut state = b.new_state();
        b.advance(&mut state, written[0]); // past the boundary marker
        let mut logits = vec![0.0; 6];
        b.boost_logits(&state, &mut logits);
        assert_eq!(
            logits[written[1]], 5.0,
            "the written form must survive untouched"
        );
    }

    #[test]
    fn test_from_phrases_latin_stays_unrepresentable_on_a_cyrillic_vocab() {
        // The honest ceiling: no amount of case folding puts Latin letters in a
        // Cyrillic-only vocabulary, so `ChatGPT` is still dropped — and the
        // warning names it so the user can write it phonetically instead.
        let tok = cyrillic_tokenizer();
        let phrases = vec![("ChatGPT".to_string(), 1.0), ("Гигаэм".to_string(), 1.0)];
        let b = Biaser::from_phrases(&tok, &phrases, 5.0).expect("one phrase compiles");
        assert_eq!(b.phrase_count(), 1);
    }

    #[test]
    fn test_from_phrases_weight_filters_per_phrase() {
        // Two representable phrases; one has weight 0 and is dropped before
        // tokenization, leaving a single compiled phrase.
        let tok = char_tokenizer();
        let phrases = vec![("аб".to_string(), 1.0), ("вг".to_string(), 0.0)];
        let b = Biaser::from_phrases(&tok, &phrases, 5.0).expect("one phrase compiles");
        assert_eq!(b.phrase_count(), 1);
    }
}