Skip to main content

snaptokens/
lib.rs

1pub mod added_tokens;
2pub mod decoders;
3pub mod json_structs;
4pub mod models;
5pub mod normalizers;
6pub mod post_processors;
7pub mod pre_tokenized;
8pub mod pre_tokenizers;
9mod tkz;
10
11use std::{borrow::Cow, fs, path::Path};
12
13use rayon::prelude::*;
14use serde_json::Value;
15
16/// Outer tasks that make document-local BPE scheduling redundant at any size.
17const WIDE_BATCH_TASKS: usize = 4;
18/// Largest input for which two or three outer tasks beat inner BPE scheduling.
19const SHORT_BATCH_INPUT_BYTES: usize = 16 * 1024;
20/// Minimum batch work per Rayon thread that can amortize outer scheduling.
21const PARALLEL_BATCH_BYTES_PER_THREAD: usize = 2 * 1024;
22/// Total work after which parallel workers use the smaller fused cache.
23const PARALLEL_CACHE_BATCH_BYTES: usize = 64 * 1024;
24/// Large ragged batches repay a tighter per-worker output reservation.
25const COMPACT_RAGGED_OUTPUT_BYTES: usize = 8 * 1024 * 1024;
26/// Work-stealing granularity for direct ragged output.
27const RAGGED_CHUNKS_PER_WORKER: usize = 8;
28/// Largest buffer that uses the conservative short-text token estimate.
29const SHORT_OUTPUT_BYTES: usize = 16 * 1024;
30
31/// Estimate token capacity without over-reserving large natural-language inputs.
32fn output_capacity(input_bytes: usize) -> usize {
33    if input_bytes <= SHORT_OUTPUT_BYTES {
34        return input_bytes.div_ceil(2).saturating_add(16).min(input_bytes);
35    }
36    input_bytes.div_ceil(3)
37}
38
39pub use self::{
40    added_tokens::{AddedTokenInfo, AddedTokens},
41    json_structs::{
42        AddedTokenConfig, DecoderConfig, ModelConfig, NormalizerConfig, PostProcessorConfig,
43        PreTokenizerConfig, TokenizerJson,
44    },
45    models::Model,
46    normalizers::{Nfc, Normalizer, Replace},
47    post_processors::PostProcessor,
48    pre_tokenizers::{ByteLevel, PreTokenizer, Split, SplitBehavior},
49};
50
51use self::{
52    added_tokens::Segment,
53    decoders::Decoder,
54    pre_tokenized::{PreTokenizedString, Split as PtSplit},
55    pre_tokenizers::FusedSplits,
56};
57
58/// Errors that can occur when constructing a [`Tokenizer`].
59#[derive(Debug, thiserror::Error)]
60pub enum Error {
61    #[error("failed to read tokenizer files: {0}")]
62    Io(#[from] std::io::Error),
63
64    #[error("failed to parse tokenizer files: {0}")]
65    Json(#[from] serde_json::Error),
66
67    #[error("invalid .tkz tokenizer: {0}")]
68    Tkz(String),
69
70    #[error("normalizer error: {0}")]
71    Normalizer(#[from] normalizers::Error),
72
73    #[error("pre-tokenizer error: {0}")]
74    PreTokenizer(#[from] pre_tokenizers::Error),
75
76    #[error("post-processor error: {0}")]
77    PostProcessor(#[from] post_processors::Error),
78
79    #[error("decoder error: {0}")]
80    Decoder(#[from] decoders::Error),
81
82    #[error("model error: {0}")]
83    Model(String),
84}
85
86/// An LLM tokenizer backed by `tokenizer.json`.
87pub struct Tokenizer {
88    added_tokens: Option<AddedTokens>,
89    normalizer: Option<Normalizer>,
90    pre_tokenizer: Option<PreTokenizer>,
91    model: Model,
92    post_processor: Option<PostProcessor>,
93    decoder: Option<Decoder>,
94    /// Whether to run vocab-aware (unbridgeable-bigram) splitting on encode.
95    /// True for metaspace models (e.g. Gemma) whose pre-tokenizer is a no-op
96    /// after normalization; false for ByteLevel models, whose regex `Split`
97    /// already produces word-level chunks. The pass is output-preserving, so
98    /// this flag only affects performance, never correctness.
99    needs_vocab_splitting: bool,
100}
101
102impl Tokenizer {
103    /// Build the pipeline steps from a parsed JSON config.
104    fn build(json: TokenizerJson) -> Result<Self, Error> {
105        let normalizer = json.normalizer.map(Normalizer::from_config).transpose()?;
106        let added_tokens =
107            AddedTokens::from_configs_with_normalizer(&json.added_tokens, normalizer.as_ref())
108                .map_err(Error::Model)?;
109        let pre_tokenizer = json
110            .pre_tokenizer
111            .map(PreTokenizer::from_config)
112            .transpose()?;
113        let model = Model::from_config(json.model).map_err(Error::Model)?;
114        let post_processor = json
115            .post_processor
116            .map(PostProcessor::from_config)
117            .transpose()?;
118        let decoder = json.decoder.map(Decoder::from_config).transpose()?;
119
120        // ByteLevel pipelines already chunk at word boundaries via their regex
121        // Split, so vocab-aware splitting adds cost without benefit. Only run it
122        // when no ByteLevel step is present (metaspace models like Gemma).
123        let needs_vocab_splitting = !pre_tokenizer
124            .as_ref()
125            .is_some_and(PreTokenizer::contains_byte_level);
126
127        Ok(Self {
128            added_tokens,
129            normalizer,
130            pre_tokenizer,
131            model,
132            post_processor,
133            decoder,
134            needs_vocab_splitting,
135        })
136    }
137
138    /// Create a tokenizer from a raw JSON value for `tokenizer.json`.
139    pub fn from_json(json: Value) -> Result<Self, Error> {
140        let json: TokenizerJson = serde_json::from_value(json)?;
141        Self::build(json)
142    }
143
144    /// Create a tokenizer from a `tokenizer.json` file.
145    pub fn from_file(path: &Path) -> Result<Self, Error> {
146        let json: TokenizerJson = serde_json::from_str(&fs::read_to_string(path)?)?;
147        Self::build(json)
148    }
149
150    /// Load a tokenizer with optional native `.tkz` sidecar persistence.
151    ///
152    /// When enabled, a JSON path creates or reuses its sibling `.tkz` file.
153    /// A direct `.tkz` path loads that file without creating another copy.
154    /// Disabling the flag is exactly equivalent to [`Self::from_file`].
155    pub fn from_file_with_tkz_cache(path: &Path, enabled: bool) -> Result<Self, Error> {
156        if !enabled {
157            return Self::from_file(path);
158        }
159        tkz::load_or_create(path)
160    }
161
162    /// Return the normalizer, if any.
163    pub fn normalizer(&self) -> Option<&Normalizer> {
164        self.normalizer.as_ref()
165    }
166
167    /// Return the pre-tokenizer, if any.
168    pub fn pre_tokenizer(&self) -> Option<&PreTokenizer> {
169        self.pre_tokenizer.as_ref()
170    }
171
172    /// Return the post-processor, if any.
173    pub fn post_processor(&self) -> Option<&PostProcessor> {
174        self.post_processor.as_ref()
175    }
176
177    /// Return the tokenization model.
178    pub fn model(&self) -> &Model {
179        &self.model
180    }
181
182    /// Return the compiled added-token set, if any.
183    pub fn added_tokens(&self) -> Option<&AddedTokens> {
184        self.added_tokens.as_ref()
185    }
186
187    /// Return the decoder, if any.
188    pub fn decoder(&self) -> Option<&Decoder> {
189        self.decoder.as_ref()
190    }
191
192    // ── Encoding ─────────────────────────────────────────────────────
193
194    /// Run the full encoding pipeline: split added tokens, normalize,
195    /// pre-tokenize, tokenize and post-process the input string.
196    pub fn encode(&self, input: &str) -> Result<Vec<u32>, Error> {
197        self.encode_with_special_tokens(input, false)
198    }
199
200    /// Run the full encoding pipeline with control over special token insertion.
201    ///
202    /// When `add_special_tokens` is true, the post-processor inserts special
203    /// tokens (e.g. BOS/EOS) as configured in the tokenizer's post-processor.
204    pub fn encode_with_special_tokens(
205        &self,
206        input: &str,
207        add_special_tokens: bool,
208    ) -> Result<Vec<u32>, Error> {
209        self.encode_with_cache_mode(input, add_special_tokens, false)
210    }
211
212    /// Run one encode with the fused-cache size selected by its batch.
213    fn encode_with_cache_mode(
214        &self,
215        input: &str,
216        add_special_tokens: bool,
217        use_parallel_cache: bool,
218    ) -> Result<Vec<u32>, Error> {
219        if input.is_empty() {
220            return if add_special_tokens {
221                Ok(self.post_process(Vec::new(), true))
222            } else {
223                Ok(Vec::new())
224            };
225        }
226
227        let fused_byte_level = self
228            .pre_tokenizer
229            .as_ref()
230            .and_then(PreTokenizer::fused_byte_level);
231        let fused_split = fused_byte_level.and_then(|(splits, _)| splits?.single());
232
233        // Standalone ByteLevel with no normalizer can scan the borrowed input
234        // directly while preserving both phases of added-token matching.
235        if let Some((None, byte_level)) = fused_byte_level
236            && self.normalizer.is_none()
237        {
238            let mut ids = Vec::with_capacity(output_capacity(input.len()));
239            self.encode_fused_byte_level_into(input, byte_level, &mut ids, use_parallel_cache)?;
240            return Ok(self.post_process(ids, add_special_tokens));
241        }
242
243        if let Some((Some(splits), _)) = fused_byte_level {
244            let mut ids = Vec::new();
245            self.encode_fused_split_into(input, splits, &mut ids, use_parallel_cache)?;
246            return Ok(self.post_process(ids, add_special_tokens));
247        }
248
249        // 1. Split on added tokens + normalize into a single buffer.
250        let (mut pts, split_applied) = self.build_pre_tokenized_for_encode(input, fused_split);
251
252        // Fused path: run only Split, then batch-tokenize with inline ByteLevel.
253        if let Some((split, byte_level)) = fused_byte_level {
254            if let Some(split) = split
255                && !split_applied
256            {
257                split.pre_tokenize(&mut pts)?;
258            }
259            byte_level.pre_tokenize_fused(&mut pts);
260            let ids = pts
261                .tokenize_batched(|buf, splits, out| {
262                    self.model.tokenize_batch_fused(buf, splits, out)
263                })
264                .map_err(Error::Model)?;
265            return Ok(self.post_process(ids, add_special_tokens));
266        }
267
268        // 2. Pre-tokenize (refine splits in place).
269        if let Some(ref pt) = self.pre_tokenizer {
270            pt.pre_tokenize(&mut pts)?;
271        }
272
273        // 2b. Break each text split at unbridgeable byte-pair boundaries.
274        //     Split at positions no vocab token can cover across adjacent
275        //     bytes. This is provably output-preserving and provides
276        //     fine-grained word-level chunking for models that don't use
277        //     ByteLevel. The table decodes byte-fallback marker spelling, so
278        //     fallback values stay in the ordinary BPE merge path.
279        if self.needs_vocab_splitting
280            && let Some(table) = self.model.bigram_bridge_table()
281        {
282            split_on_unbridgeable_bigrams(&mut pts, table);
283        }
284
285        // 3. Tokenize each text split with the model.
286        let ids = pts
287            .tokenize(|text, out| self.model.tokenize_into(text, out))
288            .map_err(Error::Model)?;
289
290        // 4. Post-process.
291        Ok(self.post_process(ids, add_special_tokens))
292    }
293
294    /// Encode a batch of inputs.
295    pub fn encode_batch<S: AsRef<str> + Sync>(
296        &self,
297        inputs: &[S],
298        add_special_tokens: bool,
299    ) -> Result<Vec<Vec<u32>>, Error> {
300        let total_bytes = inputs
301            .iter()
302            .map(|input| input.as_ref().len())
303            .sum::<usize>();
304        let parallel_bytes =
305            PARALLEL_BATCH_BYTES_PER_THREAD.saturating_mul(rayon::current_num_threads());
306        // Four vocab-split rows carry enough pipeline work to repay outer scheduling.
307        if total_bytes <= parallel_bytes
308            && (inputs.len() < WIDE_BATCH_TASKS || !self.needs_vocab_splitting)
309        {
310            return inputs
311                .iter()
312                .map(|input| self.encode_with_special_tokens(input.as_ref(), add_special_tokens))
313                .collect();
314        }
315
316        let outer_tasks = inputs.len().min(rayon::current_num_threads());
317        if outer_tasks < 2 {
318            return inputs
319                .par_iter()
320                .map(|input| self.encode_with_special_tokens(input.as_ref(), add_special_tokens))
321                .collect();
322        }
323
324        self.encode_outer_batch(inputs, add_special_tokens, outer_tasks)
325    }
326
327    /// Encode a batch into one flat token buffer plus one length per input.
328    pub fn encode_batch_ragged<S: AsRef<str> + Sync>(
329        &self,
330        inputs: &[S],
331        add_special_tokens: bool,
332    ) -> Result<(Vec<u32>, Vec<usize>), Error> {
333        let fused_byte_level = self
334            .pre_tokenizer
335            .as_ref()
336            .and_then(PreTokenizer::fused_byte_level);
337        let total_bytes = inputs
338            .iter()
339            .map(|input| input.as_ref().len())
340            .sum::<usize>();
341        let parallel_bytes =
342            PARALLEL_BATCH_BYTES_PER_THREAD.saturating_mul(rayon::current_num_threads());
343        if !add_special_tokens && let Some((Some(splits), _)) = fused_byte_level {
344            return self.encode_fused_split_ragged(inputs, splits, total_bytes, parallel_bytes);
345        }
346
347        if !add_special_tokens
348            && (inputs.len() == 1 || total_bytes <= parallel_bytes)
349            && self.normalizer.is_none()
350            && let Some((None, byte_level)) = fused_byte_level
351        {
352            return self.encode_fused_ragged_serial(inputs, byte_level, total_bytes);
353        }
354
355        if !add_special_tokens
356            && inputs.len() >= 2
357            && self.normalizer.is_none()
358            && let Some((None, byte_level)) = fused_byte_level
359        {
360            return self.encode_fused_ragged_parallel(inputs, byte_level, total_bytes);
361        }
362
363        let rows = self.encode_batch(inputs, add_special_tokens)?;
364        let total = rows.iter().map(Vec::len).sum();
365        let mut ids = Vec::with_capacity(total);
366        let mut lengths = Vec::with_capacity(rows.len());
367        for row in rows {
368            lengths.push(row.len());
369            ids.extend(row);
370        }
371        Ok((ids, lengths))
372    }
373
374    /// Encode small standalone ByteLevel batches through one fused cache stream.
375    #[inline(never)]
376    fn encode_fused_ragged_serial<S: AsRef<str> + Sync>(
377        &self,
378        inputs: &[S],
379        byte_level: &ByteLevel,
380        total_bytes: usize,
381    ) -> Result<(Vec<u32>, Vec<usize>), Error> {
382        let mut ids = Vec::with_capacity(output_capacity(total_bytes));
383        let mut lengths = Vec::with_capacity(inputs.len());
384        if self.added_tokens.as_ref().is_none_or(|added_tokens| {
385            inputs
386                .iter()
387                .all(|input| !added_tokens.has_candidate(input.as_ref()))
388        }) {
389            self.model
390                .tokenize_fused_stream("", &mut ids, false, |stream| {
391                    let mut start = 0;
392                    for input in inputs {
393                        let input = input.as_ref();
394                        byte_level.stream_fused(input, stream);
395                        let end = stream.output_len();
396                        lengths.push(end - start);
397                        start = end;
398                    }
399                })
400                .map_err(Error::Model)?;
401            return Ok((ids, lengths));
402        }
403        for input in inputs {
404            let start = ids.len();
405            self.encode_fused_byte_level_into(input.as_ref(), byte_level, &mut ids, false)?;
406            lengths.push(ids.len() - start);
407        }
408        Ok((ids, lengths))
409    }
410
411    /// Encode Split plus bulk ByteLevel rows directly into flat chunk outputs.
412    fn encode_fused_split_ragged<S: AsRef<str> + Sync>(
413        &self,
414        inputs: &[S],
415        splits: FusedSplits<'_>,
416        total_bytes: usize,
417        parallel_bytes: usize,
418    ) -> Result<(Vec<u32>, Vec<usize>), Error> {
419        let encode_chunk = |chunk: &[S]| {
420            let chunk_bytes = chunk
421                .iter()
422                .map(|input| input.as_ref().len())
423                .sum::<usize>();
424            let capacity = if total_bytes >= COMPACT_RAGGED_OUTPUT_BYTES {
425                chunk_bytes.div_ceil(2).saturating_add(16)
426            } else {
427                chunk_bytes.saturating_add(3)
428            };
429            let mut ids = Vec::with_capacity(capacity);
430            let mut lengths = Vec::with_capacity(chunk.len());
431            let use_parallel_cache = inputs.len() > 2 && total_bytes >= PARALLEL_CACHE_BATCH_BYTES;
432            if chunk
433                .iter()
434                .all(|input| self.can_encode_fused_split(input.as_ref()))
435            {
436                self.model
437                    .tokenize_fused_stream("", &mut ids, use_parallel_cache, |stream| {
438                        let mut start = 0;
439                        for input in chunk {
440                            let input = input.as_ref();
441                            let normalized = self
442                                .normalizer
443                                .as_ref()
444                                .map_or(Cow::Borrowed(input), |normalizer| {
445                                    normalizer.normalize(input)
446                                });
447                            let input = normalized.as_ref();
448                            splits.stream_into(input, stream);
449                            let end = stream.output_len();
450                            lengths.push(end - start);
451                            start = end;
452                        }
453                    })
454                    .map_err(Error::Model)?;
455                return Ok((ids, lengths));
456            }
457            for input in chunk {
458                let input = input.as_ref();
459                let start = ids.len();
460                self.encode_fused_split_into(input, splits, &mut ids, use_parallel_cache)?;
461                lengths.push(ids.len() - start);
462            }
463            Ok((ids, lengths))
464        };
465
466        if inputs.len() == 1
467            && total_bytes >= PARALLEL_CACHE_BATCH_BYTES
468            && self.normalizer.is_none()
469        {
470            let input = inputs[0].as_ref();
471            let workers = rayon::current_num_threads().min(WIDE_BATCH_TASKS);
472            if let Some(ranges) = splits.newline_partition_ranges(input, workers)
473                && ranges.len() > 1
474            {
475                let chunks = ranges
476                    .into_par_iter()
477                    .map(|range| {
478                        let input = &input[range];
479                        let mut ids = Vec::with_capacity(output_capacity(input.len()));
480                        self.model
481                            .tokenize_fused_stream("", &mut ids, true, |stream| {
482                                splits.stream_into(input, stream);
483                            })
484                            .map_err(Error::Model)?;
485                        Ok(ids)
486                    })
487                    .collect::<Result<Vec<_>, Error>>()?;
488                let token_count = chunks.iter().map(Vec::len).sum();
489                let mut ids = Vec::with_capacity(token_count);
490                for chunk in chunks {
491                    ids.extend(chunk);
492                }
493                return Ok((ids, vec![token_count]));
494            }
495        }
496
497        // One or two rows reuse one hot local cache instead of duplicating its working set.
498        if inputs.len() < 3 || total_bytes <= parallel_bytes {
499            return encode_chunk(inputs);
500        }
501
502        let workers = inputs.len().min(rayon::current_num_threads());
503        let chunk_size = inputs.len().div_ceil(workers * RAGGED_CHUNKS_PER_WORKER);
504        let chunks = inputs
505            .par_chunks(chunk_size)
506            .map(encode_chunk)
507            .collect::<Result<Vec<_>, Error>>()?;
508        let token_count = chunks.iter().map(|(ids, _)| ids.len()).sum();
509        let mut ids = Vec::with_capacity(token_count);
510        let mut lengths = Vec::with_capacity(inputs.len());
511        for (chunk_ids, chunk_lengths) in chunks {
512            ids.extend(chunk_ids);
513            lengths.extend(chunk_lengths);
514        }
515        Ok((ids, lengths))
516    }
517
518    /// Encode standalone ByteLevel rows into coarse flat worker chunks.
519    #[inline(never)]
520    fn encode_fused_ragged_parallel<S: AsRef<str> + Sync>(
521        &self,
522        inputs: &[S],
523        byte_level: &ByteLevel,
524        total_bytes: usize,
525    ) -> Result<(Vec<u32>, Vec<usize>), Error> {
526        let workers = inputs.len().min(rayon::current_num_threads());
527        let chunk_size = inputs.len().div_ceil(workers * RAGGED_CHUNKS_PER_WORKER);
528        let use_parallel_cache = total_bytes >= PARALLEL_CACHE_BATCH_BYTES;
529        let chunks = inputs
530            .par_chunks(chunk_size)
531            .map(|chunk| {
532                let chunk_bytes = chunk
533                    .iter()
534                    .map(|input| input.as_ref().len())
535                    .sum::<usize>();
536                let mut ids = Vec::with_capacity(output_capacity(chunk_bytes));
537                let mut lengths = Vec::with_capacity(chunk.len());
538                for input in chunk {
539                    let start = ids.len();
540                    self.encode_fused_byte_level_into(
541                        input.as_ref(),
542                        byte_level,
543                        &mut ids,
544                        use_parallel_cache,
545                    )?;
546                    lengths.push(ids.len() - start);
547                }
548                Ok((ids, lengths))
549            })
550            .collect::<Result<Vec<_>, Error>>()?;
551
552        let token_count = chunks.iter().map(|(ids, _)| ids.len()).sum();
553        let mut ids = Vec::with_capacity(token_count);
554        let mut lengths = Vec::with_capacity(inputs.len());
555        for (chunk_ids, chunk_lengths) in chunks {
556            ids.extend(chunk_ids);
557            lengths.extend(chunk_lengths);
558        }
559        Ok((ids, lengths))
560    }
561
562    /// Encode outer-parallel work out of line to keep the narrow hot path compact.
563    #[inline(never)]
564    fn encode_outer_batch<S: AsRef<str> + Sync>(
565        &self,
566        inputs: &[S],
567        add_special_tokens: bool,
568        outer_tasks: usize,
569    ) -> Result<Vec<Vec<u32>>, Error> {
570        let use_parallel_cache = inputs
571            .iter()
572            .map(|input| input.as_ref().len())
573            .sum::<usize>()
574            >= PARALLEL_CACHE_BATCH_BYTES;
575        inputs
576            .par_iter()
577            .map(|input| {
578                let input = input.as_ref();
579                // Wide batches occupy the cores directly; smaller batches do
580                // so only when cross-pool scheduling costs more than the work.
581                if outer_tasks >= WIDE_BATCH_TASKS || input.len() <= SHORT_BATCH_INPUT_BYTES {
582                    pre_tokenized::without_inner_parallelism(|| {
583                        self.encode_with_cache_mode(input, add_special_tokens, use_parallel_cache)
584                    })
585                } else {
586                    self.encode_with_cache_mode(input, add_special_tokens, use_parallel_cache)
587                }
588            })
589            .collect()
590    }
591
592    /// Append one directly fused standalone ByteLevel encoding to `ids`.
593    fn encode_fused_byte_level_into(
594        &self,
595        input: &str,
596        byte_level: &ByteLevel,
597        ids: &mut Vec<u32>,
598        use_parallel_cache: bool,
599    ) -> Result<(), Error> {
600        if let Some(added_tokens) = &self.added_tokens
601            && added_tokens.has_candidate(input)
602        {
603            for segment in added_tokens.split(input) {
604                match segment {
605                    Segment::Token(id) => ids.push(id),
606                    Segment::Text(text) => self
607                        .model
608                        .tokenize_fused_stream(text, ids, use_parallel_cache, |stream| {
609                            byte_level.stream_fused(text, stream)
610                        })
611                        .map_err(Error::Model)?,
612                }
613            }
614            return Ok(());
615        }
616
617        self.model
618            .tokenize_fused_stream(input, ids, use_parallel_cache, |stream| {
619                byte_level.stream_fused(input, stream)
620            })
621            .map_err(Error::Model)
622    }
623
624    /// Append a normalized Split plus bulk ByteLevel row without materializing splits.
625    fn encode_fused_split_into(
626        &self,
627        input: &str,
628        splits: FusedSplits<'_>,
629        ids: &mut Vec<u32>,
630        use_parallel_cache: bool,
631    ) -> Result<(), Error> {
632        if !self.can_encode_fused_split(input) {
633            let segments = self.segment_input(input);
634            let normalized_added_tokens = self
635                .added_tokens
636                .as_ref()
637                .filter(|added_tokens| added_tokens.has_normalized());
638            self.model
639                .tokenize_fused_stream(input, ids, use_parallel_cache, |stream| {
640                    for segment in segments {
641                        match segment {
642                            Segment::Token(id) => stream.push_id(id),
643                            Segment::Text(text) => {
644                                let normalized = self
645                                    .normalizer
646                                    .as_ref()
647                                    .map_or(Cow::Borrowed(text), |normalizer| {
648                                        normalizer.normalize(text)
649                                    });
650                                if let Some(added_tokens) = normalized_added_tokens {
651                                    for segment in
652                                        added_tokens.split_normalized(normalized.as_ref())
653                                    {
654                                        match segment {
655                                            Segment::Token(id) => stream.push_id(id),
656                                            Segment::Text(text) => {
657                                                splits.stream_into(text, stream);
658                                                stream.flush_pending();
659                                            }
660                                        }
661                                    }
662                                } else {
663                                    let text = normalized.as_ref();
664                                    splits.stream_into(text, stream);
665                                    stream.flush_pending();
666                                }
667                            }
668                        }
669                    }
670                })
671                .map_err(Error::Model)?;
672            return Ok(());
673        }
674
675        let normalized = self
676            .normalizer
677            .as_ref()
678            .map_or(Cow::Borrowed(input), |normalizer| {
679                normalizer.normalize(input)
680            });
681        let input = normalized.as_ref();
682        self.model
683            .tokenize_fused_stream(input, ids, use_parallel_cache, |stream| {
684                splits.stream_into(input, stream)
685            })
686            .map_err(Error::Model)?;
687        Ok(())
688    }
689
690    /// Return whether added-token matching permits whole-row normalization.
691    fn can_encode_fused_split(&self, input: &str) -> bool {
692        !self
693            .added_tokens
694            .as_ref()
695            .is_some_and(|added_tokens| added_tokens.has_candidate(input))
696    }
697
698    /// Replace the post-processor.
699    pub fn set_post_processor(&mut self, pp: Option<PostProcessor>) {
700        self.post_processor = pp;
701    }
702
703    /// Replace the normalizer.
704    pub fn set_normalizer(&mut self, normalizer: Option<Normalizer>) {
705        self.normalizer = normalizer;
706    }
707
708    pub fn post_process(&self, ids: Vec<u32>, add_special_tokens: bool) -> Vec<u32> {
709        match &self.post_processor {
710            Some(pp) => pp.post_process_single(ids, add_special_tokens),
711            None => ids,
712        }
713    }
714
715    // ── Decoding ─────────────────────────────────────────────────────
716
717    /// Decode token IDs back into text.
718    ///
719    /// If `skip_special_tokens` is true, added tokens marked as special
720    /// are omitted from the output.
721    pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
722        let mut tokens = Vec::with_capacity(ids.len());
723        for &id in ids {
724            if skip_special_tokens
725                && let Some(ref at) = self.added_tokens
726                && at.is_special(id)
727            {
728                continue;
729            }
730            // Match HuggingFace behavior: silently skip unknown IDs (e.g.
731            // models like Qwen3-0.6B-FP8 emit IDs in the gap between
732            // tokenizer.json's vocab and the embedding matrix). Erroring
733            // here would kill streaming generation on a single bad token.
734            if let Some(token_str) = self.id_to_token(id) {
735                tokens.push(token_str.to_string());
736            }
737        }
738
739        self.decode_tokens(tokens)
740    }
741
742    /// Decode a sequence of token strings back into text.
743    ///
744    /// Applies the decoder pipeline (e.g. ByteLevel → convert "Ġ" back to " ")
745    /// without going through the ID→string lookup.  When no decoder is
746    /// configured the tokens are concatenated with no separator.
747    pub fn decode_tokens(&self, tokens: Vec<String>) -> Result<String, Error> {
748        match &self.decoder {
749            Some(dec) => dec.decode(tokens).map_err(Error::Decoder),
750            None => Ok(tokens.concat()),
751        }
752    }
753
754    /// Decode a batch of token ID sequences.
755    pub fn decode_batch(
756        &self,
757        sentences: &[&[u32]],
758        skip_special_tokens: bool,
759    ) -> Result<Vec<String>, Error> {
760        sentences
761            .iter()
762            .map(|ids| self.decode(ids, skip_special_tokens))
763            .collect()
764    }
765
766    // ── Vocabulary access ────────────────────────────────────────────
767
768    /// Look up the string for a token ID, checking added tokens first,
769    /// then the model vocabulary.
770    pub fn id_to_token(&self, id: u32) -> Option<&str> {
771        if let Some(ref at) = self.added_tokens
772            && let Some(s) = at.id_to_token(id)
773        {
774            return Some(s);
775        }
776        self.model.id_to_token(id)
777    }
778
779    /// Look up the token ID for a string.
780    ///
781    /// Added tokens are checked first (they shadow any BPE model entry with
782    /// the same string), then the BPE model vocabulary.
783    pub fn token_to_id(&self, token: &str) -> Option<u32> {
784        if let Some(ref at) = self.added_tokens
785            && let Some(id) = at.token_to_id(token)
786        {
787            return Some(id);
788        }
789        self.model.token_to_id(token)
790    }
791
792    /// Return the vocabulary size (model tokens + added tokens).
793    pub fn vocab_size(&self) -> usize {
794        let model_size = self.model.vocab_size();
795        let added_size = self.added_tokens.as_ref().map_or(0, |at| at.len());
796        model_size + added_size
797    }
798
799    /// Return whether this token ID is marked special in the added-token set.
800    pub fn is_special_token(&self, id: u32) -> bool {
801        self.added_tokens
802            .as_ref()
803            .is_some_and(|added_tokens| added_tokens.is_special(id))
804    }
805
806    // ── Internal helpers ─────────────────────────────────────────────
807
808    /// Build a [`PreTokenizedString`] by splitting on added tokens and
809    /// normalizing text segments into a single contiguous buffer.
810    pub fn build_pre_tokenized(&self, input: &str) -> PreTokenizedString {
811        let segments = self.segment_input(input);
812        self.build_pre_tokenized_from_segments(input, &segments)
813    }
814
815    /// Split input around configured added tokens without changing text segments.
816    fn segment_input<'a>(&self, input: &'a str) -> Vec<Segment<'a>> {
817        match &self.added_tokens {
818            Some(at) => at.split_non_normalized(input),
819            None => vec![Segment::Text(input)],
820        }
821    }
822
823    /// Build encode input, fusing the fixed Split scan with the ASCII NFC proof when possible.
824    fn build_pre_tokenized_for_encode(
825        &self,
826        input: &str,
827        fused_split: Option<&Split>,
828    ) -> (PreTokenizedString, bool) {
829        let segments = self.segment_input(input);
830        if matches!(self.normalizer, Some(Normalizer::Nfc(_)))
831            && !self
832                .added_tokens
833                .as_ref()
834                .is_some_and(AddedTokens::has_normalized)
835            && let Some(split) = fused_split
836            && split.supports_ascii_nfc_fusion()
837        {
838            let capacity = input.len().div_ceil(4).max(segments.len());
839            let mut splits = Vec::with_capacity(capacity);
840            let mut buffer_len = 0;
841            let mut is_ascii = true;
842
843            for segment in &segments {
844                match segment {
845                    Segment::Token(id) => splits.push(PtSplit {
846                        range: buffer_len..buffer_len,
847                        token_id: Some(*id),
848                    }),
849                    Segment::Text(text) => {
850                        if !split.append_ascii_splits(text, buffer_len, &mut splits) {
851                            is_ascii = false;
852                            break;
853                        }
854                        buffer_len += text.len();
855                    }
856                }
857            }
858
859            if is_ascii {
860                let mut buffer = String::with_capacity(buffer_len);
861                for segment in &segments {
862                    if let Segment::Text(text) = segment {
863                        buffer.push_str(text);
864                    }
865                }
866                return (PreTokenizedString::new(buffer, splits), true);
867            }
868        }
869
870        (
871            self.build_pre_tokenized_from_segments(input, &segments),
872            false,
873        )
874    }
875
876    /// Normalize pre-segmented text into the single-buffer tokenizer representation.
877    fn build_pre_tokenized_from_segments(
878        &self,
879        input: &str,
880        segments: &[Segment<'_>],
881    ) -> PreTokenizedString {
882        let normalized_added_tokens = self
883            .added_tokens
884            .as_ref()
885            .filter(|added_tokens| added_tokens.has_normalized());
886
887        // Fast path: if there's exactly one Text segment (no added token matches)
888        // and normalization returns Cow::Borrowed, we just need a string copy.
889        if segments.len() == 1
890            && let Segment::Text(text) = segments[0]
891            && normalized_added_tokens.is_none()
892        {
893            let normalized = match &self.normalizer {
894                Some(n) => n.normalize(text),
895                None => std::borrow::Cow::Borrowed(text),
896            };
897            return match normalized {
898                std::borrow::Cow::Borrowed(_) => PreTokenizedString::from_text(text),
899                std::borrow::Cow::Owned(s) => {
900                    let len = s.len();
901                    PreTokenizedString::new(
902                        s,
903                        vec![PtSplit {
904                            range: 0..len,
905                            token_id: None,
906                        }],
907                    )
908                }
909            };
910        }
911
912        let mut buffer = String::with_capacity(input.len());
913        let mut splits = Vec::new();
914
915        for seg in segments {
916            match seg {
917                Segment::Token(id) => {
918                    let start = buffer.len();
919                    splits.push(PtSplit {
920                        range: start..start,
921                        token_id: Some(*id),
922                    });
923                }
924                Segment::Text(text) => {
925                    if text.is_empty() {
926                        continue;
927                    }
928                    let normalized = match &self.normalizer {
929                        Some(n) => n.normalize(text),
930                        None => std::borrow::Cow::Borrowed(*text),
931                    };
932                    if let Some(added_tokens) = normalized_added_tokens {
933                        // Normalized tokens only inspect spans left unmatched by
934                        // the raw-token phase, exactly as Hugging Face does.
935                        for segment in added_tokens.split_normalized(&normalized) {
936                            let start = buffer.len();
937                            match segment {
938                                Segment::Token(id) => splits.push(PtSplit {
939                                    range: start..start,
940                                    token_id: Some(id),
941                                }),
942                                Segment::Text(text) => {
943                                    buffer.push_str(text);
944                                    splits.push(PtSplit {
945                                        range: start..buffer.len(),
946                                        token_id: None,
947                                    });
948                                }
949                            }
950                        }
951                    } else {
952                        let start = buffer.len();
953                        buffer.push_str(&normalized);
954                        splits.push(PtSplit {
955                            range: start..buffer.len(),
956                            token_id: None,
957                        });
958                    }
959                }
960            }
961        }
962
963        PreTokenizedString::new(buffer, splits)
964    }
965}
966
967/// Split each text chunk at unbridgeable byte-pair boundaries using the
968/// vocab-derived bigram bridge table.
969///
970/// A byte pair (prev, cur) is "unbridgeable" if no vocabulary token can cover
971/// that adjacent input sequence. Splitting at such boundaries is provably
972/// output-preserving: any BPE merge that spans the boundary would produce a
973/// token covering that byte pair, which cannot exist in the vocabulary.
974///
975/// This generalizes newline splitting and enables fine-grained word-level
976/// chunking even in metaspace tokenizers like Gemma, where the pre-tokenizer
977/// is effectively a no-op after normalization.
978fn split_on_unbridgeable_bigrams(
979    pts: &mut PreTokenizedString,
980    bigram_table: &models::bpe::BigramBridgeTable,
981) {
982    let bytes = pts.buffer().as_bytes();
983    let mut new_splits = Vec::with_capacity(pts.splits().len() * 2);
984
985    for split in pts.splits() {
986        if split.token_id.is_some() || split.range.is_empty() {
987            new_splits.push(split.clone());
988            continue;
989        }
990
991        let end = split.range.end;
992        let mut start = split.range.start;
993
994        for i in (start + 1)..end {
995            let prev = bytes[i - 1];
996            let cur = bytes[i];
997
998            // Split here if:
999            // 1. No vocab token can cover this byte pair, AND
1000            // 2. Position i is a UTF-8 char boundary (cur is not a continuation byte)
1001            if !bigram_table.is_bridgeable(prev, cur) && (cur & 0xC0) != 0x80 {
1002                new_splits.push(PtSplit {
1003                    range: start..i,
1004                    token_id: None,
1005                });
1006                start = i;
1007            }
1008        }
1009
1010        // Push the final segment
1011        new_splits.push(PtSplit {
1012            range: start..end,
1013            token_id: None,
1014        });
1015    }
1016
1017    pts.refine_splits(new_splits);
1018}
1019
1020// ---------------------------------------------------------------------------
1021// Streaming decode
1022// ---------------------------------------------------------------------------
1023
1024/// Stateful incremental decoder.
1025///
1026/// Wraps the sliding-window state needed by [`decode_stream_step`] so callers
1027/// don't have to manage `ids`, `prefix`, and `prefix_index` themselves.
1028pub struct DecodeStream {
1029    skip_special_tokens: bool,
1030    ids: Vec<u32>,
1031    prefix: String,
1032    prefix_index: usize,
1033}
1034
1035impl DecodeStream {
1036    pub fn new(ids: Vec<u32>, skip_special_tokens: bool) -> Self {
1037        Self {
1038            skip_special_tokens,
1039            ids,
1040            prefix: String::new(),
1041            prefix_index: 0,
1042        }
1043    }
1044
1045    pub fn step(
1046        &mut self,
1047        tokenizer: &Tokenizer,
1048        token_ids: Vec<u32>,
1049    ) -> Result<Option<String>, String> {
1050        decode_stream_step(
1051            tokenizer,
1052            token_ids,
1053            self.skip_special_tokens,
1054            &mut self.ids,
1055            &mut self.prefix,
1056            &mut self.prefix_index,
1057        )
1058    }
1059}
1060
1061/// Advance an incremental decode stream by one or more token IDs.
1062///
1063/// Maintains a sliding window in `ids` and a `prefix` string to subtract,
1064/// emitting text chunks as soon as enough context is available.
1065/// Incomplete UTF-8 (signalled by U+FFFD in the decoder output) is held back
1066/// until a subsequent token resolves it.
1067///
1068/// # Arguments
1069/// * `token_ids` — new token IDs to append
1070/// * `skip_special_tokens` — whether to omit special tokens from the output
1071/// * `ids` — mutable buffer of all IDs decoded so far (updated in place)
1072/// * `prefix` — previously returned text, subtracted to yield the next chunk
1073/// * `prefix_index` — index in `ids` where the current prefix window starts
1074///
1075/// # Returns
1076/// `Ok(Some(chunk))` when new text is available, `Ok(None)` when more tokens
1077/// are needed, `Err(msg)` if the decoder produces output inconsistent with the
1078/// stored prefix (should be treated as a stream-reset signal).
1079pub fn decode_stream_step(
1080    tokenizer: &Tokenizer,
1081    token_ids: Vec<u32>,
1082    skip_special_tokens: bool,
1083    ids: &mut Vec<u32>,
1084    prefix: &mut String,
1085    prefix_index: &mut usize,
1086) -> Result<Option<String>, String> {
1087    const REPLACEMENT: char = '\u{FFFD}';
1088
1089    // If the prefix is empty but we already have buffered IDs (e.g. seeded
1090    // with prompt tokens), prime the prefix before adding the new token.
1091    if prefix.is_empty() && !ids.is_empty() {
1092        let s = tokenizer
1093            .decode(ids, skip_special_tokens)
1094            .map_err(|e| e.to_string())?;
1095        if !s.ends_with(REPLACEMENT) {
1096            *prefix = s;
1097            *prefix_index = ids.len();
1098        }
1099    }
1100
1101    ids.extend(token_ids);
1102
1103    let string = tokenizer
1104        .decode(ids, skip_special_tokens)
1105        .map_err(|e| e.to_string())?;
1106
1107    if string.len() > prefix.len() && !string.ends_with(REPLACEMENT) {
1108        if !string.starts_with(prefix.as_str()) {
1109            return Err(format!(
1110                "Invalid prefix encountered while decoding stream. \
1111                 Expected prefix: '{}', Actual string: '{}'",
1112                prefix, string,
1113            ));
1114        }
1115        let new_text = string[prefix.len()..].to_string();
1116        // Slide the active decode window without replacing its allocation.
1117        drop(ids.drain(..*prefix_index));
1118        *prefix = tokenizer
1119            .decode(ids, skip_special_tokens)
1120            .map_err(|e| e.to_string())?;
1121        *prefix_index = ids.len();
1122        Ok(Some(new_text))
1123    } else {
1124        Ok(None)
1125    }
1126}
1127
1128#[cfg(test)]
1129mod tests {
1130    use std::path::PathBuf;
1131
1132    use super::*;
1133
1134    const HF_MODELS: &[&str] = &[
1135        "Qwen/Qwen3-0.6B",
1136        "zai-org/GLM-4.7",
1137        "deepseek-ai/DeepSeek-V3.2",
1138        "MiniMaxAI/MiniMax-M2.1",
1139        "openai/gpt-oss-120b",
1140        "mistralai/Mistral-Nemo-Instruct-2407",
1141        "Qwen/Qwen3-235B-A22B-Instruct-2507",
1142        "Qwen/Qwen3-Coder-480B-A35B-Instruct",
1143        "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
1144        "nvidia/Qwen3-Nemotron-235B-A22B-GenRM",
1145        "hoangquan456/Kimi-K2.5",
1146    ];
1147
1148    /// Resolves a remote test fixture to the cached local `tokenizer.json`.
1149    fn tokenizer_json_path(model: &str) -> anyhow::Result<PathBuf> {
1150        let api = hf_hub::api::sync::Api::new()?;
1151        Ok(api.model(model.to_string()).get("tokenizer.json")?)
1152    }
1153
1154    /// Loads fastokens from a test fixture's local `tokenizer.json`.
1155    fn load_tokenizer(model: &str) -> anyhow::Result<Tokenizer> {
1156        let path = tokenizer_json_path(model)?;
1157        Ok(Tokenizer::from_file(&path)?)
1158    }
1159
1160    /// Loads the reference tokenizer from the same local JSON test fixture.
1161    fn load_reference_tokenizer(model: &str) -> anyhow::Result<tokenizers::Tokenizer> {
1162        let path = tokenizer_json_path(model)?;
1163        tokenizers::Tokenizer::from_file(path).map_err(|error| anyhow::anyhow!(error))
1164    }
1165
1166    /// Verifies that supported Hugging Face tokenizer JSON files deserialize.
1167    #[test]
1168    fn parse_hf_json() {
1169        for model in HF_MODELS {
1170            let json_path = tokenizer_json_path(model).unwrap_or_else(|e| panic!("{model}: {e}"));
1171            let _: TokenizerJson = serde_json::from_str(&fs::read_to_string(json_path).unwrap())
1172                .unwrap_or_else(|e| panic!("{model}: {e}"));
1173        }
1174    }
1175
1176    /// Verify that encode_batch matches sequential encodes.
1177    #[test]
1178    fn encode_batch_matches_sequential() {
1179        let model = "MiniMaxAI/MiniMax-M2.1";
1180        let ours = load_tokenizer(model).unwrap();
1181
1182        let inputs = &["Hello, world!", "The quick brown fox", "Test", ""];
1183        let batch_results = ours.encode_batch(inputs, false).unwrap();
1184
1185        for (input, batch_result) in inputs.iter().zip(&batch_results) {
1186            let sequential_result = ours.encode(input).unwrap();
1187            assert_eq!(
1188                batch_result, &sequential_result,
1189                "batch mismatch for {input:?}"
1190            );
1191        }
1192    }
1193
1194    /// Verify ragged output boundaries across text, added-token, and empty rows.
1195    #[test]
1196    fn encode_batch_ragged_matches_nested() {
1197        let ours = load_tokenizer("openai-community/gpt2").unwrap();
1198        let mut inputs = vec!["Hello, world! ".repeat(32); 64];
1199        inputs[1] = "<|endoftext|>".into();
1200        inputs[2] = String::new();
1201        let expected = ours.encode_batch(&inputs, false).unwrap();
1202        let (actual_ids, actual_lengths) = ours.encode_batch_ragged(&inputs, false).unwrap();
1203
1204        let expected_lengths = expected.iter().map(Vec::len).collect::<Vec<_>>();
1205        let expected_ids = expected.into_iter().flatten().collect::<Vec<_>>();
1206        assert_eq!(actual_lengths, expected_lengths);
1207        assert_eq!(actual_ids, expected_ids);
1208    }
1209
1210    /// Preserve BPE merges between byte-fallback expansions of adjacent Unicode characters.
1211    #[test]
1212    fn byte_fallback_merge_crosses_unicode_boundary() {
1213        let tokenizer = Tokenizer::from_json(serde_json::json!({
1214            "normalizer": null,
1215            "pre_tokenizer": null,
1216            "model": {
1217                "type": "BPE",
1218                "vocab": {
1219                    "<unk>": 0,
1220                    "<0xC3>": 1,
1221                    "<0xA9>": 2,
1222                    "<0xAA>": 3,
1223                    "<0xA9><0xC3>": 4
1224                },
1225                "merges": [["<0xA9>", "<0xC3>"]],
1226                "unk_token": "<unk>",
1227                "byte_fallback": true
1228            },
1229            "post_processor": null,
1230            "decoder": null
1231        }))
1232        .unwrap();
1233
1234        assert_eq!(tokenizer.encode("éê").unwrap(), vec![1, 4, 3]);
1235    }
1236
1237    /// Match normalized tokens only after raw-only tokens have been protected.
1238    #[test]
1239    fn normalized_added_tokens_follow_the_two_phase_pipeline() {
1240        let tokenizer = Tokenizer::from_json(serde_json::json!({
1241            "added_tokens": [
1242                {
1243                    "id": 10,
1244                    "content": "e\u{301}!",
1245                    "single_word": false,
1246                    "lstrip": false,
1247                    "rstrip": false,
1248                    "normalized": true,
1249                    "special": false
1250                },
1251                {
1252                    "id": 11,
1253                    "content": "e\u{301}?",
1254                    "single_word": false,
1255                    "lstrip": false,
1256                    "rstrip": false,
1257                    "normalized": false,
1258                    "special": false
1259                }
1260            ],
1261            "normalizer": { "type": "NFC" },
1262            "pre_tokenizer": null,
1263            "model": {
1264                "type": "BPE",
1265                "vocab": { "<unk>": 0 },
1266                "merges": [],
1267                "unk_token": "<unk>",
1268                "byte_fallback": false
1269            },
1270            "post_processor": null,
1271            "decoder": null
1272        }))
1273        .unwrap();
1274
1275        assert_eq!(tokenizer.encode("é!").unwrap(), vec![10]);
1276        assert_eq!(tokenizer.encode("e\u{301}!").unwrap(), vec![10]);
1277        assert_eq!(tokenizer.encode("e\u{301}?").unwrap(), vec![11]);
1278    }
1279
1280    /// Verify that vocab access methods work correctly.
1281    #[test]
1282    fn vocab_access() {
1283        let model = "MiniMaxAI/MiniMax-M2.1";
1284        let ours = load_tokenizer(model).unwrap();
1285
1286        assert!(ours.vocab_size() > 0);
1287
1288        let token_str = ours.id_to_token(0).expect("token 0 should exist");
1289        let id = ours
1290            .token_to_id(token_str)
1291            .expect("reverse lookup should work");
1292        assert_eq!(id, 0);
1293    }
1294
1295    #[test]
1296    fn public_added_token_accessors_expose_added_vocab() {
1297        let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
1298        let added_tokens = tok.added_tokens().expect("expected added tokens");
1299
1300        let think_id = tok.token_to_id("<think>").expect("<think> should exist");
1301        assert_eq!(added_tokens.token_to_id("<think>"), Some(think_id));
1302        assert_eq!(added_tokens.id_to_token(think_id), Some("<think>"));
1303
1304        let mut entries: Vec<_> = added_tokens.iter().collect();
1305        entries.sort_by_key(|entry| entry.id);
1306        let special_entry = entries
1307            .iter()
1308            .find(|entry| entry.special)
1309            .expect("expected at least one special added token");
1310        assert!(tok.is_special_token(special_entry.id));
1311        assert!(
1312            entries
1313                .iter()
1314                .any(|entry| entry.id == think_id && entry.content == "<think>"),
1315            "added-token iterator should expose <think>"
1316        );
1317    }
1318
1319    // ── Correctness tests against HuggingFace tokenizers ─────────────
1320
1321    /// Comprehensive corpus of inputs designed to exercise tokenizer edge
1322    /// cases. Used by the multi-model correctness tests below.
1323    const CORPUS: &[&str] = &[
1324        // ── empty / trivial ──
1325        "",
1326        " ",
1327        "  ",
1328        "\n",
1329        "\t",
1330        "\r\n",
1331        // ── single characters ──
1332        "a",
1333        "Z",
1334        "0",
1335        "!",
1336        "\u{00e9}", // é (precomposed)
1337        "\u{4e2d}", // 中
1338        // ── basic text ──
1339        "Hello, world!",
1340        "The quick brown fox jumps over the lazy dog.",
1341        "A short sentence.",
1342        // ── whitespace variations ──
1343        "  leading spaces",
1344        "trailing spaces  ",
1345        "  both  sides  ",
1346        "multiple    internal    spaces",
1347        "tabs\there\tand\tthere",
1348        "line\none\nline\ntwo",
1349        "windows\r\nline\r\nendings",
1350        "mixed\n\ttabs and\r\nnewlines  with  spaces",
1351        // ── numbers ──
1352        "42",
1353        "3.14159",
1354        "1,000,000",
1355        "0xFF",
1356        "1e-10",
1357        "Numbers 1234567890 and mixed ABC123def",
1358        // ── punctuation / special characters ──
1359        "Hello!!! How are you???",
1360        "@user #hashtag $100 %50 ^caret &amp *star",
1361        "a-b_c.d,e;f:g",
1362        "(parentheses) [brackets] {braces}",
1363        "\"double quotes\" 'single quotes' `backticks`",
1364        "path/to/file.txt",
1365        "https://example.com/path?q=test&lang=en#section",
1366        "Special chars: @#$%^&*()_+-=[]{}|;':\",./<>?",
1367        // ── Unicode: Latin accented ──
1368        "caf\u{00e9} r\u{00e9}sum\u{00e9} na\u{00ef}ve",
1369        "\u{00fc}ber stra\u{00df}e gr\u{00f6}\u{00df}e",
1370        "se\u{00f1}or ni\u{00f1}o a\u{00f1}o",
1371        // ── Unicode: CJK ──
1372        "\u{4f60}\u{597d}\u{4e16}\u{754c}",         // 你好世界
1373        "\u{3053}\u{3093}\u{306b}\u{3061}\u{306f}", // こんにちは
1374        "\u{c548}\u{b155}\u{d558}\u{c138}\u{c694}", // 안녕하세요
1375        // ── Unicode: Cyrillic ──
1376        "\u{041f}\u{0440}\u{0438}\u{0432}\u{0435}\u{0442} \u{043c}\u{0438}\u{0440}",
1377        // ── Unicode: Arabic ──
1378        "\u{0645}\u{0631}\u{062d}\u{0628}\u{0627}",
1379        // ── Unicode: Devanagari ──
1380        "\u{0928}\u{092e}\u{0938}\u{094d}\u{0924}\u{0947}",
1381        // ── Unicode: Emoji ──
1382        "\u{1f600}\u{1f680}\u{2764}\u{fe0f}",
1383        "\u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467}\u{200d}\u{1f466}",
1384        "\u{1f1fa}\u{1f1f8}", // 🇺🇸
1385        // ── Unicode: combining marks (NFD forms) ──
1386        "e\u{0301}", // e + combining acute
1387        "n\u{0303}", // n + combining tilde
1388        "a\u{0308}", // a + combining diaeresis
1389        // ── mixed scripts ──
1390        "Hello \u{4e16}\u{754c} \u{041c}\u{0438}\u{0440}!",
1391        "User123 wrote: \u{4f60}\u{597d}!",
1392        // ── code / programming ──
1393        "fn main() { println!(\"hello\"); }",
1394        "def foo(x: int) -> str:\n    return str(x)",
1395        "SELECT * FROM users WHERE id = 1;",
1396        "if (x > 0 && y < 10) { z = x + y; }",
1397        "<html><body><p>Hello</p></body></html>",
1398        "#include <stdio.h>\nint main() { return 0; }",
1399        "import numpy as np\nx = np.array([1, 2, 3])",
1400        // ── JSON / structured data ──
1401        "{\"key\": \"value\", \"number\": 42, \"array\": [1, 2, 3]}",
1402        "[{\"id\": 1}, {\"id\": 2}]",
1403        // ── repeated patterns ──
1404        "aaaaaaaaaa",
1405        "abababababababab",
1406        "the the the the the the the the",
1407        "....",
1408        "----",
1409        "    ",
1410        "\n\n\n\n",
1411        // ── longer mixed content ──
1412        "This is a longer sentence with various elements: numbers (42, 3.14), \
1413         symbols (@#$), Unicode (caf\u{00e9}, \u{4f60}\u{597d}), and more.",
1414        "The year 2024 was notable for advances in AI. Models like GPT-4 and \
1415         Claude demonstrated remarkable capabilities in reasoning, coding, and \
1416         multilingual understanding.",
1417        // ── alphabet / character sequences ──
1418        "a b c d e f g h i j k l m n o p q r s t u v w x y z",
1419        "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
1420        "0123456789",
1421        // ── boundary / edge cases ──
1422        "a\nb\nc\n",
1423        "# Heading\n\n- item 1\n- item 2\n\n```code```",
1424        "\u{ffff}",  // max BMP non-character
1425        "\u{0080}",  // first non-ASCII
1426        "\u{07ff}",  // max 2-byte UTF-8
1427        "\u{0800}",  // first 3-byte UTF-8
1428        "\u{10000}", // first surrogate-pair range
1429        // ── unusual / invalid-ish Unicode ──
1430        "\u{fffd}",                                  // replacement character
1431        "\u{feff}Hello",                             // BOM prefix
1432        "\u{0000}",                                  // null
1433        "abc\u{0000}def",                            // embedded null
1434        "\u{fffe}",                                  // non-character
1435        "\u{fdd0}",                                  // non-character (FDD0 block)
1436        "\u{200b}\u{200c}\u{200d}",                  // zero-width space / ZWNJ / ZWJ
1437        "\u{202e}Hello\u{202c}",                     // RTL override + pop directional
1438        "\u{0001}\u{0002}\u{001f}\u{007f}",          // C0 controls + DEL
1439        "\u{0300}",                                  // lone combining grave (no base)
1440        "a\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}", // 5 combining marks on one base
1441        "\u{e000}\u{f8ff}",                          // private use area
1442        "\u{01c5}\u{01c8}\u{01cb}",                  // titlecase letters (Dž Lj Nj)
1443        "\u{2028}\u{2029}",                          // line / paragraph separators
1444        "\u{fff9}\u{fffa}\u{fffb}",                  // interlinear annotation
1445        "\u{d7ff}\u{10ffff}",                        // last before surrogates + max codepoint
1446        // ── potential BPE merge edge cases ──
1447        "ab",
1448        "abc",
1449        "abcd",
1450        "aaa",
1451        "aaaa",
1452        "aaaaa",
1453        // ── markdown / formatting ──
1454        "**bold** *italic* ~~strikethrough~~ __underline__",
1455        "```rust\nfn main() {}\n```",
1456        "> blockquote\n>> nested",
1457        "| col1 | col2 |\n|------|------|\n| a    | b    |",
1458    ];
1459
1460    /// Helper: compare both encoding and decoding of every input in `corpus`
1461    /// between our tokenizer and the HuggingFace tokenizer for a given model.
1462    /// Returns a list of failure descriptions (empty = all passed).
1463    fn compare_encode_decode(model_name: &str, corpus: &[&str]) -> Vec<String> {
1464        let hf = load_reference_tokenizer(model_name)
1465            .unwrap_or_else(|e| panic!("{model_name}: HF load failed: {e}"));
1466        let ours = load_tokenizer(model_name)
1467            .unwrap_or_else(|e| panic!("{model_name}: fastokens load failed: {e}"));
1468
1469        let mut failures = Vec::new();
1470        for &input in corpus {
1471            let hf_enc = hf
1472                .encode(input, false)
1473                .unwrap_or_else(|e| panic!("{model_name}: HF encode({input:?}): {e}"));
1474            let hf_ids = hf_enc.get_ids().to_vec();
1475            let our_ids = match ours.encode(input) {
1476                Ok(ids) => ids,
1477                Err(e) => {
1478                    failures.push(format!("  encode error on {input:?}: {e}"));
1479                    continue;
1480                }
1481            };
1482            if our_ids != hf_ids {
1483                failures.push(format!(
1484                    "  encode mismatch on {input:?}: got {} tokens, expected {}\n\
1485                     \x20   ours: {:?}\n\
1486                     \x20   hf:   {:?}",
1487                    our_ids.len(),
1488                    hf_ids.len(),
1489                    &our_ids[..our_ids.len().min(20)],
1490                    &hf_ids[..hf_ids.len().min(20)],
1491                ));
1492            }
1493
1494            // Decode comparison (skip empty inputs / empty token sequences).
1495            if input.is_empty() || hf_ids.is_empty() {
1496                continue;
1497            }
1498            let hf_decoded = match hf.decode(&hf_ids, false) {
1499                Ok(d) => d,
1500                Err(_) => continue,
1501            };
1502            let our_decoded = match ours.decode(&hf_ids, false) {
1503                Ok(d) => d,
1504                Err(e) => {
1505                    failures.push(format!("  decode error on {input:?}: {e}"));
1506                    continue;
1507                }
1508            };
1509            if our_decoded != hf_decoded {
1510                failures.push(format!(
1511                    "  decode mismatch on {input:?}:\n\
1512                     \x20   ours: {:?}\n\
1513                     \x20   hf:   {:?}",
1514                    &our_decoded[..our_decoded.len().min(100)],
1515                    &hf_decoded[..hf_decoded.len().min(100)],
1516                ));
1517            }
1518        }
1519        failures
1520    }
1521
1522    // ── Per-model encoding correctness ───────────────────────────────
1523
1524    #[test]
1525    fn correctness_gpt2() {
1526        let f = compare_encode_decode("openai-community/gpt2", CORPUS);
1527        assert!(f.is_empty(), "openai-community/gpt2:\n{}", f.join("\n"));
1528    }
1529
1530    #[test]
1531    fn correctness_minimax_m2_1() {
1532        let f = compare_encode_decode("MiniMaxAI/MiniMax-M2.1", CORPUS);
1533        assert!(f.is_empty(), "MiniMaxAI/MiniMax-M2.1:\n{}", f.join("\n"));
1534    }
1535
1536    #[test]
1537    fn correctness_nemotron() {
1538        let f = compare_encode_decode("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", CORPUS);
1539        assert!(
1540            f.is_empty(),
1541            "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16:\n{}",
1542            f.join("\n")
1543        );
1544    }
1545
1546    #[test]
1547    fn correctness_deepseek_v3_2() {
1548        let f = compare_encode_decode("deepseek-ai/DeepSeek-V3.2", CORPUS);
1549        assert!(f.is_empty(), "deepseek-ai/DeepSeek-V3.2:\n{}", f.join("\n"));
1550    }
1551
1552    #[test]
1553    fn correctness_gpt_oss() {
1554        let f = compare_encode_decode("openai/gpt-oss-120b", CORPUS);
1555        assert!(f.is_empty(), "openai/gpt-oss-120b:\n{}", f.join("\n"));
1556    }
1557
1558    /// Verifies exact-token lookup uses the original piece and only when merges are ignored.
1559    #[test]
1560    fn ignore_merges_preserves_piece_semantics() {
1561        let tokenizer = |ignore_merges| {
1562            Tokenizer::from_json(serde_json::json!({
1563                "normalizer": null,
1564                "pre_tokenizer": null,
1565                "model": {
1566                    "type": "BPE",
1567                    "vocab": {"a": 0, "b": 1, "c": 2, "d": 3, "ab": 4, "abc": 5},
1568                    "merges": [],
1569                    "ignore_merges": ignore_merges
1570                },
1571                "post_processor": null,
1572                "decoder": null
1573            }))
1574            .unwrap()
1575        };
1576
1577        let merged = tokenizer(false);
1578        assert_eq!(merged.encode("ab").unwrap(), vec![0, 1]);
1579
1580        let ignored = tokenizer(true);
1581        assert_eq!(ignored.encode("abc").unwrap(), vec![5]);
1582        assert_eq!(ignored.encode("abd").unwrap(), vec![0, 1, 3]);
1583    }
1584
1585    #[test]
1586    fn ignore_merges_glm47() {
1587        let model = "zai-org/GLM-4.7";
1588        let hf = load_reference_tokenizer(model).unwrap();
1589        let ours = load_tokenizer(model).unwrap();
1590
1591        // " имущества" is a single token (140507) in GLM-4.7 vocab.
1592        // BPE merging alone produces 3 tokens — ignore_merges must
1593        // short-circuit to the vocab entry.
1594        let text = " имущества";
1595        let hf_ids = hf.encode(text, false).unwrap().get_ids().to_vec();
1596        let our_ids = ours.encode(text).unwrap();
1597        assert_eq!(
1598            our_ids, hf_ids,
1599            "ignore_merges mismatch on {text:?}: ours={our_ids:?} hf={hf_ids:?}"
1600        );
1601
1602        // Also test with random-token-decoded text (the benchmark pattern).
1603        let vocab_size = hf.get_vocab_size(false) as u64;
1604        let random_ids: Vec<u32> = (0..5000)
1605            .map(|i| {
1606                ((i as u64).wrapping_mul(6364136223846793005).wrapping_add(1) % vocab_size) as u32
1607            })
1608            .collect();
1609        let text = hf.decode(&random_ids, true).unwrap();
1610        let hf_enc = hf.encode(text.as_str(), false).unwrap().get_ids().to_vec();
1611        let our_enc = ours.encode(&text).unwrap();
1612        assert_eq!(
1613            our_enc,
1614            hf_enc,
1615            "ignore_merges random-decode mismatch: {} vs {} tokens",
1616            our_enc.len(),
1617            hf_enc.len()
1618        );
1619    }
1620
1621    #[test]
1622    fn correctness_qwen3() {
1623        let f = compare_encode_decode("Qwen/Qwen3-0.6B", CORPUS);
1624        assert!(f.is_empty(), "Qwen/Qwen3-0.6B:\n{}", f.join("\n"));
1625    }
1626
1627    #[test]
1628    fn correctness_mistral_nemo() {
1629        let f = compare_encode_decode("mistralai/Mistral-Nemo-Instruct-2407", CORPUS);
1630        assert!(
1631            f.is_empty(),
1632            "mistralai/Mistral-Nemo-Instruct-2407:\n{}",
1633            f.join("\n")
1634        );
1635    }
1636
1637    #[test]
1638    fn correctness_qwen3_nemotron() {
1639        let f = compare_encode_decode("nvidia/Qwen3-Nemotron-235B-A22B-GenRM", CORPUS);
1640        assert!(
1641            f.is_empty(),
1642            "nvidia/Qwen3-Nemotron-235B-A22B-GenRM:\n{}",
1643            f.join("\n")
1644        );
1645    }
1646
1647    #[test]
1648    fn correctness_kimi_k2_5() {
1649        let f = compare_encode_decode("hoangquan456/Kimi-K2.5", CORPUS);
1650        assert!(f.is_empty(), "hoangquan456/Kimi-K2.5:\n{}", f.join("\n"));
1651    }
1652
1653    // ── Cache consistency ────────────────────────────────────────────
1654
1655    /// Verify that encoding the same input twice produces identical results,
1656    /// exercising both the cold (cache miss) and warm (cache hit) paths.
1657    #[test]
1658    fn cache_consistency() {
1659        let model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16";
1660        let ours = load_tokenizer(model).unwrap();
1661
1662        let inputs = &[
1663            "Hello, world!",
1664            "The quick brown fox jumps over the lazy dog.",
1665            "caf\u{00e9} r\u{00e9}sum\u{00e9}",
1666            "\u{4f60}\u{597d}\u{4e16}\u{754c}",
1667            "fn main() { println!(\"hello\"); }",
1668            "a b c d e f g h i j k l m n o p",
1669            "aaaaaaaaaa bbbbbbbbbb cccccccccc",
1670        ];
1671
1672        for &input in inputs {
1673            let first = ours.encode(input).unwrap();
1674            let second = ours.encode(input).unwrap();
1675            assert_eq!(first, second, "cache inconsistency for {input:?}");
1676            // Third call to exercise potential L1→L2 promotion paths.
1677            let third = ours.encode(input).unwrap();
1678            assert_eq!(first, third, "cache inconsistency (3rd call) for {input:?}");
1679        }
1680    }
1681
1682    /// Same as above but for the fused byte-level path (Nemotron uses
1683    /// Sequence([Split, ByteLevel]) which triggers the fused code path).
1684    #[test]
1685    fn cache_consistency_fused() {
1686        let model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16";
1687        let ours = load_tokenizer(model).unwrap();
1688
1689        // Verify the fused path is active.
1690        assert!(
1691            ours.pre_tokenizer
1692                .as_ref()
1693                .and_then(PreTokenizer::fused_byte_level)
1694                .is_some(),
1695            "expected fused path for {model}"
1696        );
1697
1698        // Run the same input many times to stress the fused cache.
1699        let input = "The year 2024 was notable for advances in AI. Models like \
1700                      GPT-4 and Claude demonstrated remarkable capabilities.";
1701        let baseline = ours.encode(input).unwrap();
1702        for i in 0..20 {
1703            let result = ours.encode(input).unwrap();
1704            assert_eq!(result, baseline, "fused cache drift on iteration {i}");
1705        }
1706    }
1707
1708    // ── Added tokens (model-specific) ────────────────────────────────
1709
1710    /// MiniMax-M2.1 has added tokens like <filename>, <reponame>, <think>,
1711    /// etc. Verify they are handled identically to HF.
1712    #[test]
1713    fn added_tokens_minimax() {
1714        let corpus = &[
1715            "<filename>",
1716            "open <filename> for reading",
1717            "<filename><reponame>",
1718            "printf(\"%s <filename>\\n\")",
1719            "<think>Let me reason about this.</think>",
1720            "<think>load <filename> from <reponame></think>",
1721            "<file> is not <filename>",
1722            "<fim_prefix>code here<fim_suffix>more code<fim_middle>",
1723        ];
1724        let f = compare_encode_decode("MiniMaxAI/MiniMax-M2.1", corpus);
1725        assert!(
1726            f.is_empty(),
1727            "MiniMaxAI/MiniMax-M2.1 added tokens:\n{}",
1728            f.join("\n")
1729        );
1730    }
1731
1732    /// DeepSeek-V3.2 added tokens.
1733    #[test]
1734    fn added_tokens_deepseek() {
1735        let corpus = &[
1736            "<|begin▁of▁sentence|>Hello",
1737            "Hello<|end▁of▁sentence|>",
1738            "<|User|>What is 2+2?<|Assistant|>4<|end▁of▁sentence|>",
1739            "Normal text without special tokens",
1740            "<|tool▁calls▁begin|>call<|tool▁calls▁end|>",
1741        ];
1742        let f = compare_encode_decode("deepseek-ai/DeepSeek-V3.2", corpus);
1743        assert!(
1744            f.is_empty(),
1745            "deepseek-ai/DeepSeek-V3.2 added tokens:\n{}",
1746            f.join("\n")
1747        );
1748    }
1749
1750    /// Qwen3 added tokens.
1751    #[test]
1752    fn added_tokens_qwen3() {
1753        let corpus = &[
1754            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>",
1755            "<|im_start|>user\nHello!<|im_end|>",
1756            "<|endoftext|>",
1757            "Plain text with no special tokens at all.",
1758        ];
1759        let f = compare_encode_decode("Qwen/Qwen3-0.6B", corpus);
1760        assert!(
1761            f.is_empty(),
1762            "Qwen/Qwen3-0.6B added tokens:\n{}",
1763            f.join("\n")
1764        );
1765    }
1766
1767    /// token_to_id must find added tokens, not just BPE model vocab entries.
1768    ///
1769    /// Root cause of the Qwen3VLProcessor._check_special_mm_tokens failure:
1770    /// `convert_tokens_to_ids("<|image_pad|>")` calls `token_to_id`, which
1771    /// previously only searched the BPE model vocabulary and returned None for
1772    /// added tokens, causing the processor to compare input_ids against
1773    /// unk_token_id (0) instead of the real image-pad token ID.
1774    #[test]
1775    fn token_to_id_searches_added_tokens() {
1776        let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
1777        // These tokens live in added_tokens, not the BPE model vocab.
1778        for token in &[
1779            "<|image_pad|>",
1780            "<|vision_start|>",
1781            "<|vision_end|>",
1782            "<|im_start|>",
1783        ] {
1784            let id = tok.token_to_id(token);
1785            assert!(id.is_some(), "token_to_id({token:?}) returned None");
1786            // Round-trip: the ID must decode back to the same string.
1787            assert_eq!(tok.id_to_token(id.unwrap()), Some(*token));
1788        }
1789    }
1790
1791    /// Qwen3-VL vision tokens — the exact text that triggered:
1792    ///
1793    ///   ValueError: Failed to apply Qwen3VLProcessor on
1794    ///   data={'text': '<|vision_start|><|image_pad|><|vision_end|>'}
1795    ///   with kwargs={'truncation': False}
1796    ///
1797    /// Qwen3-0.6B ships with the full set of VL tokens in its added_tokens
1798    /// array.  A sequence that consists *entirely* of adjacent special tokens
1799    /// (no regular text in between) exercises the code path where
1800    /// build_pre_tokenized produces only zero-length Token splits.
1801    #[test]
1802    fn added_tokens_qwen3vl_vision_sequence() {
1803        let corpus = &[
1804            // Exact failing input from vLLM / Qwen3VLProcessor.
1805            "<|vision_start|><|image_pad|><|vision_end|>",
1806            // Bare image-pad token.
1807            "<|image_pad|>",
1808            // Multiple adjacent image-pad tokens (real prompts have dozens).
1809            "<|vision_start|><|image_pad|><|image_pad|><|image_pad|><|image_pad|><|vision_end|>",
1810            // Mixed: VL tokens followed by regular text.
1811            "<|vision_start|><|image_pad|><|vision_end|>\nDescribe this image.",
1812        ];
1813        let f = compare_encode_decode("Qwen/Qwen3.5-27B", corpus);
1814        assert!(
1815            f.is_empty(),
1816            "Qwen/Qwen3.5-27B VL vision sequence:\n{}",
1817            f.join("\n")
1818        );
1819    }
1820
1821    /// Nemotron added tokens.
1822    #[test]
1823    fn added_tokens_nemotron() {
1824        let corpus = &[
1825            "<|begin_of_text|>Hello world",
1826            "Hello<|end_of_text|>",
1827            "<|start_header_id|>system<|end_header_id|>\n\nYou are helpful.<|eot_id|>",
1828            "<|start_header_id|>user<|end_header_id|>\n\nHi!<|eot_id|>",
1829            "No special tokens here.",
1830        ];
1831        let f = compare_encode_decode("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", corpus);
1832        assert!(
1833            f.is_empty(),
1834            "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 added tokens:\n{}",
1835            f.join("\n")
1836        );
1837    }
1838
1839    // ── Long input stress test ───────────────────────────────────────
1840
1841    /// Verify correctness on a longer input that exercises the parallel
1842    /// tokenization path (>128 splits).
1843    #[test]
1844    fn long_input_correctness() {
1845        let model_name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16";
1846        let hf = load_reference_tokenizer(model_name).unwrap();
1847        let ours = load_tokenizer(model_name).unwrap();
1848
1849        // Build a ~10KB input from repeated varied content.
1850        let block = "The quick brown fox jumps over the lazy dog. \
1851                      Numbers: 42, 3.14, 1000. Code: fn main() {} \
1852                      Unicode: caf\u{00e9}, \u{4f60}\u{597d}. \
1853                      Special: @#$%^&*(). ";
1854        let input: String = block.repeat(100);
1855        assert!(input.len() > 8000);
1856
1857        let hf_ids = hf.encode(input.as_str(), false).unwrap().get_ids().to_vec();
1858        let our_ids = ours.encode(&input).unwrap();
1859        assert_eq!(
1860            our_ids,
1861            hf_ids,
1862            "long input mismatch: {} vs {} tokens",
1863            our_ids.len(),
1864            hf_ids.len(),
1865        );
1866    }
1867
1868    /// Match sequential encoding when one newline-rich row takes the parallel fused path.
1869    #[test]
1870    fn newline_partitioned_ragged_matches_sequential() {
1871        let ours = load_tokenizer("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16").unwrap();
1872        assert!(ours.normalizer.is_none());
1873        for input in [
1874            "Text and numbers 123. Unicode: café, 你好. Punctuation?!\r\n\n".repeat(1_200),
1875            "one long line with no partition boundary ".repeat(2_000),
1876        ] {
1877            assert!(input.len() >= PARALLEL_CACHE_BATCH_BYTES);
1878            let expected = ours.encode(&input).unwrap();
1879            let (actual, lengths) = ours.encode_batch_ragged(&[input.as_str()], false).unwrap();
1880            assert_eq!(lengths, [expected.len()]);
1881            assert_eq!(actual, expected);
1882        }
1883    }
1884
1885    /// Same long-input test for a non-fused model.
1886    #[test]
1887    fn long_input_correctness_minimax() {
1888        let model_name = "MiniMaxAI/MiniMax-M2.1";
1889        let hf = load_reference_tokenizer(model_name).unwrap();
1890        let ours = load_tokenizer(model_name).unwrap();
1891
1892        let block = "The quick brown fox jumps over the lazy dog. \
1893                      Numbers: 42, 3.14, 1000. Code: fn main() {} \
1894                      Unicode: caf\u{00e9}, \u{4f60}\u{597d}. \
1895                      Special: @#$%^&*(). ";
1896        let input: String = block.repeat(100);
1897
1898        let hf_ids = hf.encode(input.as_str(), false).unwrap().get_ids().to_vec();
1899        let our_ids = ours.encode(&input).unwrap();
1900        assert_eq!(
1901            our_ids,
1902            hf_ids,
1903            "long input mismatch: {} vs {} tokens",
1904            our_ids.len(),
1905            hf_ids.len(),
1906        );
1907    }
1908
1909    // ── Extended dataset tests (run with `cargo test -- --ignored`) ──
1910
1911    use std::sync::OnceLock;
1912
1913    struct ExtendedCorpus {
1914        longbench: Vec<String>,
1915        sharegpt: Vec<String>,
1916    }
1917
1918    fn extended_corpus() -> &'static ExtendedCorpus {
1919        static CORPUS: OnceLock<ExtendedCorpus> = OnceLock::new();
1920        CORPUS.get_or_init(|| {
1921            let api = hf_hub::api::sync::Api::new().unwrap();
1922
1923            // LongBench-v2: first 100 samples
1924            let lb_repo = api.dataset("zai-org/LongBench-v2".to_string());
1925            let lb_path = lb_repo.get("data.json").unwrap();
1926            let lb_data: Vec<serde_json::Value> =
1927                serde_json::from_str(&fs::read_to_string(lb_path).unwrap()).unwrap();
1928            let longbench: Vec<String> = lb_data
1929                .iter()
1930                .filter_map(|item| {
1931                    let ctx = item.get("context")?.as_str()?;
1932                    if ctx.is_empty() {
1933                        None
1934                    } else {
1935                        Some(ctx.to_string())
1936                    }
1937                })
1938                .take(100)
1939                .collect();
1940
1941            // ShareGPT52K: first 1000 samples
1942            let sg_repo = api.dataset("RyokoAI/ShareGPT52K".to_string());
1943            let sg_path = sg_repo.get("sg_90k_part1.json").unwrap();
1944            let sg_data: Vec<serde_json::Value> =
1945                serde_json::from_str(&fs::read_to_string(sg_path).unwrap()).unwrap();
1946            let sharegpt: Vec<String> = sg_data
1947                .iter()
1948                .filter_map(|item| {
1949                    let messages = item.get("conversations")?.as_array()?;
1950                    let parts: Vec<String> = messages
1951                        .iter()
1952                        .filter_map(|msg| {
1953                            let role = msg
1954                                .get("from")
1955                                .and_then(|v| v.as_str())
1956                                .unwrap_or("unknown");
1957                            let value = msg.get("value").and_then(|v| v.as_str())?;
1958                            if value.is_empty() {
1959                                return None;
1960                            }
1961                            Some(format!("[{role}]: {value}"))
1962                        })
1963                        .collect();
1964                    if parts.is_empty() {
1965                        None
1966                    } else {
1967                        Some(parts.join("\n\n"))
1968                    }
1969                })
1970                .take(1_000)
1971                .collect();
1972
1973            ExtendedCorpus {
1974                longbench,
1975                sharegpt,
1976            }
1977        })
1978    }
1979
1980    /// Compare encoding and decoding in batches using encode_batch.
1981    fn compare_encode_decode_batched(
1982        model_name: &str,
1983        corpus: &[String],
1984        batch_size: usize,
1985        progress: bool,
1986    ) -> Vec<String> {
1987        let hf = load_reference_tokenizer(model_name)
1988            .unwrap_or_else(|e| panic!("{model_name}: HF load failed: {e}"));
1989        let ours = load_tokenizer(model_name)
1990            .unwrap_or_else(|e| panic!("{model_name}: fastokens load failed: {e}"));
1991
1992        let total = corpus.len();
1993        let mut processed = 0usize;
1994        let mut failures = Vec::new();
1995        for chunk in corpus.chunks(batch_size) {
1996            let hf_results: Vec<Vec<u32>> = chunk
1997                .iter()
1998                .map(|input| {
1999                    hf.encode(input.as_str(), false)
2000                        .unwrap_or_else(|e| panic!("{model_name}: HF encode: {e}"))
2001                        .get_ids()
2002                        .to_vec()
2003                })
2004                .collect();
2005
2006            let our_results = match ours.encode_batch(chunk, false) {
2007                Ok(r) => r,
2008                Err(e) => {
2009                    failures.push(format!("  encode_batch error: {e}"));
2010                    continue;
2011                }
2012            };
2013
2014            for (i, (hf_ids, our_ids)) in hf_results.iter().zip(our_results.iter()).enumerate() {
2015                let input = &chunk[i];
2016                let input_preview = {
2017                    let mut end = input.len().min(80);
2018                    while end < input.len() && !input.is_char_boundary(end) {
2019                        end += 1;
2020                    }
2021                    &input[..end]
2022                };
2023
2024                if our_ids != hf_ids {
2025                    failures.push(format!(
2026                        "  encode mismatch on {:?}: got {} tokens, expected {}\n\
2027                         \x20   ours: {:?}\n\
2028                         \x20   hf:   {:?}",
2029                        input_preview,
2030                        our_ids.len(),
2031                        hf_ids.len(),
2032                        &our_ids[..our_ids.len().min(20)],
2033                        &hf_ids[..hf_ids.len().min(20)],
2034                    ));
2035                }
2036
2037                // Decode comparison.
2038                if hf_ids.is_empty() || input.is_empty() {
2039                    continue;
2040                }
2041                let hf_decoded = match hf.decode(hf_ids, false) {
2042                    Ok(d) => d,
2043                    Err(_) => continue,
2044                };
2045                let our_decoded = match ours.decode(hf_ids, false) {
2046                    Ok(d) => d,
2047                    Err(e) => {
2048                        failures.push(format!("  decode error on {input_preview:?}: {e}"));
2049                        continue;
2050                    }
2051                };
2052                if our_decoded != hf_decoded {
2053                    failures.push(format!(
2054                        "  decode mismatch on {input_preview:?}:\n\
2055                         \x20   ours: {:?}\n\
2056                         \x20   hf:   {:?}",
2057                        &our_decoded[..our_decoded.len().min(100)],
2058                        &hf_decoded[..hf_decoded.len().min(100)],
2059                    ));
2060                }
2061            }
2062            processed += chunk.len();
2063            if progress {
2064                eprint!(
2065                    "\r  {model_name}: {processed}/{total} ({:.0}%)",
2066                    processed as f64 / total as f64 * 100.0,
2067                );
2068            }
2069        }
2070        if progress {
2071            eprintln!();
2072        }
2073        failures
2074    }
2075
2076    fn run_extended(model_name: &str) {
2077        let progress = std::env::var("EXTENDED_PROGRESS").is_ok();
2078        let corpus = extended_corpus();
2079        if progress {
2080            eprintln!(
2081                "  {model_name}: longbench ({} samples)",
2082                corpus.longbench.len()
2083            );
2084        }
2085        let mut failures =
2086            compare_encode_decode_batched(model_name, &corpus.longbench, 10, progress);
2087        if progress {
2088            eprintln!(
2089                "  {model_name}: sharegpt ({} samples)",
2090                corpus.sharegpt.len()
2091            );
2092        }
2093        failures.extend(compare_encode_decode_batched(
2094            model_name,
2095            &corpus.sharegpt,
2096            10,
2097            progress,
2098        ));
2099        assert!(
2100            failures.is_empty(),
2101            "{model_name} extended ({} failures):\n{}",
2102            failures.len(),
2103            failures.join("\n"),
2104        );
2105    }
2106
2107    #[test]
2108    #[ignore]
2109    fn extended_minimax_m2_1() {
2110        run_extended("MiniMaxAI/MiniMax-M2.1");
2111    }
2112
2113    #[test]
2114    #[ignore]
2115    fn extended_nemotron() {
2116        run_extended("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16");
2117    }
2118
2119    #[test]
2120    #[ignore]
2121    fn extended_deepseek_v3_2() {
2122        run_extended("deepseek-ai/DeepSeek-V3.2");
2123    }
2124
2125    #[test]
2126    #[ignore]
2127    fn extended_gpt_oss() {
2128        run_extended("openai/gpt-oss-120b");
2129    }
2130
2131    #[test]
2132    #[ignore]
2133    fn extended_qwen3() {
2134        run_extended("Qwen/Qwen3-0.6B");
2135    }
2136
2137    #[test]
2138    #[ignore]
2139    fn extended_mistral_nemo() {
2140        run_extended("mistralai/Mistral-Nemo-Instruct-2407");
2141    }
2142
2143    #[test]
2144    #[ignore]
2145    fn extended_qwen3_nemotron() {
2146        run_extended("nvidia/Qwen3-Nemotron-235B-A22B-GenRM");
2147    }
2148
2149    #[test]
2150    #[ignore]
2151    fn extended_mistral_large() {
2152        run_extended("mistralai/Mistral-Large-3-675B-Instruct-2512");
2153    }
2154
2155    #[test]
2156    #[ignore]
2157    fn extended_qwen_small() {
2158        run_extended("Qwen/Qwen3-0.6B");
2159    }
2160
2161    // ── encode / decode correctness ─────────────────────────────────────────
2162
2163    /// Encode without special tokens → decode → original text, for all models.
2164    #[test]
2165    fn encode_decode_roundtrip_all_models() {
2166        let texts = &[
2167            "Hello, world!",
2168            "日本語テスト",
2169            "The quick brown fox jumps over the lazy dog.",
2170            "fn main() { println!(\"hello\"); }",
2171            "   leading and trailing spaces   ",
2172            "line1\nline2\ttabbed",
2173            "0123456789",
2174            "🌍🎉✨",
2175        ];
2176        let failures: Vec<String> = HF_MODELS
2177            .iter()
2178            .flat_map(|model| {
2179                let tok = match load_tokenizer(model) {
2180                    Ok(t) => t,
2181                    Err(e) => return vec![format!("{model}: load error: {e}")],
2182                };
2183                texts
2184                    .iter()
2185                    .filter_map(|text| {
2186                        let ids = tok.encode_with_special_tokens(text, false).ok()?;
2187                        let decoded = tok.decode(&ids, false).ok()?;
2188                        if decoded != *text {
2189                            Some(format!("{model}: {text:?} → {decoded:?}"))
2190                        } else {
2191                            None
2192                        }
2193                    })
2194                    .collect()
2195            })
2196            .collect();
2197        assert!(
2198            failures.is_empty(),
2199            "encode→decode roundtrip failures:\n{}",
2200            failures.join("\n")
2201        );
2202    }
2203
2204    /// Models with add_bos_token=true prepend BOS when add_special_tokens=true.
2205    ///
2206    /// In HuggingFace, `add_bos_token` in `tokenizer_config.json` gates whether
2207    /// the BOS token is inserted. Our Rust side implements this through the
2208    /// post-processor configured in `tokenizer.json`.  This test verifies the
2209    /// three key behaviours:
2210    ///
2211    /// 1. add_special_tokens=true  → BOS is the first token ID
2212    /// 2. add_special_tokens=false → BOS is absent
2213    /// 3. A model without a BOS post-processor (Qwen3) never adds BOS
2214    #[test]
2215    fn add_bos_token() {
2216        // ── model WITH add_bos_token (Mistral-Nemo, BOS = <s> id=1) ──────────
2217        let tok = load_tokenizer("mistralai/Mistral-Nemo-Instruct-2407").unwrap();
2218        let bos_id = tok.token_to_id("<s>").expect("<s> not in vocabulary");
2219
2220        let with_bos = tok.encode_with_special_tokens("hello world", true).unwrap();
2221        let without_bos = tok
2222            .encode_with_special_tokens("hello world", false)
2223            .unwrap();
2224
2225        assert_eq!(
2226            with_bos.first().copied(),
2227            Some(bos_id),
2228            "first token should be BOS when add_special_tokens=true"
2229        );
2230        assert_ne!(
2231            without_bos.first().copied(),
2232            Some(bos_id),
2233            "BOS should be absent when add_special_tokens=false"
2234        );
2235        // The content tokens are identical in both cases.
2236        assert_eq!(&with_bos[1..], without_bos.as_slice());
2237
2238        // ── model WITHOUT add_bos_token (Qwen3-0.6B) ─────────────────────────
2239        let tok_q = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2240        let with_flag = tok_q
2241            .encode_with_special_tokens("hello world", true)
2242            .unwrap();
2243        let without_flag = tok_q
2244            .encode_with_special_tokens("hello world", false)
2245            .unwrap();
2246        assert_eq!(
2247            with_flag, without_flag,
2248            "Qwen3 has no BOS post-processor — add_special_tokens should have no effect"
2249        );
2250    }
2251
2252    /// decode(ids, skip=true) omits BOS/EOS; decode(ids, skip=false) includes them.
2253    #[test]
2254    fn decode_skip_special_tokens() {
2255        // Mistral-Nemo adds BOS (<s>, id=1) in basic encoding.
2256        let model = "mistralai/Mistral-Nemo-Instruct-2407";
2257        let tok = load_tokenizer(model).unwrap();
2258        let text = "hello world";
2259        let ids_with = tok.encode_with_special_tokens(text, true).unwrap();
2260        let ids_without = tok.encode_with_special_tokens(text, false).unwrap();
2261        assert!(
2262            ids_with.len() > ids_without.len(),
2263            "expected BOS/EOS from {model}"
2264        );
2265
2266        let skipped = tok.decode(&ids_with, true).unwrap();
2267        assert_eq!(skipped, text);
2268
2269        let full = tok.decode(&ids_with, false).unwrap();
2270        assert_ne!(full, text);
2271        assert!(full.contains(text));
2272    }
2273
2274    /// decode_batch produces the same results as sequential decode.
2275    #[test]
2276    fn decode_batch_matches_sequential() {
2277        let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2278        let sentences = &["first sentence", "second sentence", "日本語テスト", ""];
2279        let id_batches: Vec<Vec<u32>> = sentences
2280            .iter()
2281            .map(|s| tok.encode_with_special_tokens(s, false).unwrap())
2282            .collect();
2283        let refs: Vec<&[u32]> = id_batches.iter().map(Vec::as_slice).collect();
2284        let batch_out = tok.decode_batch(&refs, false).unwrap();
2285        for (out, expected) in batch_out.iter().zip(sentences.iter()) {
2286            assert_eq!(out, expected);
2287        }
2288    }
2289
2290    /// decode_tokens(strings) == decode(ids) for the same sequence.
2291    #[test]
2292    fn decode_tokens_matches_decode_by_id() {
2293        let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2294        for text in &["Hello, world!", "The quick brown fox", "🌍 emoji"] {
2295            let ids = tok.encode_with_special_tokens(text, false).unwrap();
2296            let token_strings: Vec<String> = ids
2297                .iter()
2298                .map(|&id| tok.id_to_token(id).unwrap().to_string())
2299                .collect();
2300            let via_ids = tok.decode(&ids, false).unwrap();
2301            let via_tokens = tok.decode_tokens(token_strings).unwrap();
2302            assert_eq!(via_ids, via_tokens, "mismatch for {text:?}");
2303        }
2304    }
2305
2306    /// Encoding an empty string produces an empty token list.
2307    #[test]
2308    fn empty_string_encode_decode() {
2309        let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2310        let ids = tok.encode_with_special_tokens("", false).unwrap();
2311        assert!(ids.is_empty(), "expected no tokens for empty string");
2312        assert_eq!(tok.decode(&[], false).unwrap(), "");
2313    }
2314
2315    /// encode → decode → encode is stable (idempotent on second encode).
2316    #[test]
2317    fn encode_is_stable_after_decode() {
2318        let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2319        for text in &["hello world", "日本語テスト", "fn foo() {}"] {
2320            let ids1 = tok.encode_with_special_tokens(text, false).unwrap();
2321            let decoded = tok.decode(&ids1, false).unwrap();
2322            let ids2 = tok.encode_with_special_tokens(&decoded, false).unwrap();
2323            assert_eq!(ids1, ids2, "encode not stable after decode for {text:?}");
2324        }
2325    }
2326
2327    /// post_process with add_special_tokens=false is the identity for all models.
2328    #[test]
2329    fn post_process_false_is_identity_all_models() {
2330        for model in HF_MODELS {
2331            let tok = load_tokenizer(model).unwrap();
2332            let payload = vec![100u32, 200, 300];
2333            let out = tok.post_process(payload.clone(), false);
2334            assert_eq!(
2335                out, payload,
2336                "{model}: post_process(false) should be identity"
2337            );
2338        }
2339    }
2340
2341    /// post_process(true) adds at least as many tokens as post_process(false).
2342    #[test]
2343    fn post_process_true_adds_special_tokens() {
2344        // Use Mistral-Nemo which has a post-processor that adds BOS.
2345        let tok = load_tokenizer("mistralai/Mistral-Nemo-Instruct-2407").unwrap();
2346        let payload = vec![10u32, 20, 30];
2347        let without = tok.post_process(payload.clone(), false);
2348        let with_sp = tok.post_process(payload.clone(), true);
2349        assert_eq!(without, payload);
2350        assert!(
2351            with_sp.len() > without.len(),
2352            "expected special tokens to be added"
2353        );
2354        // The original payload IDs appear contiguously somewhere in the output.
2355        assert!(
2356            with_sp
2357                .windows(payload.len())
2358                .any(|w| w == payload.as_slice()),
2359            "payload should appear contiguously in post-processed output"
2360        );
2361    }
2362
2363    /// decode of an unknown ID silently skips it, matching HuggingFace.
2364    #[test]
2365    fn decode_unknown_id_is_skipped() {
2366        let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2367        assert_eq!(tok.decode(&[u32::MAX], false).unwrap(), "");
2368    }
2369
2370    /// decode interleaves valid tokens with unknown IDs, dropping only the bad ones.
2371    #[test]
2372    fn decode_mixed_valid_and_unknown_ids() {
2373        let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2374        let valid = tok.encode_with_special_tokens("hello", false).unwrap();
2375        let mut mixed = valid.clone();
2376        mixed.push(u32::MAX);
2377        mixed.extend(tok.encode_with_special_tokens(" world", false).unwrap());
2378        let expected = tok.decode(&valid, false).unwrap()
2379            + &tok
2380                .decode(
2381                    &tok.encode_with_special_tokens(" world", false).unwrap(),
2382                    false,
2383                )
2384                .unwrap();
2385        assert_eq!(tok.decode(&mixed, false).unwrap(), expected);
2386    }
2387
2388    /// id_to_token / token_to_id round-trip for sampled IDs across all models.
2389    #[test]
2390    fn token_id_roundtrip_all_models() {
2391        let probe_ids = [0u32, 1, 2, 100, 1000, 10_000];
2392        let failures: Vec<String> = HF_MODELS
2393            .iter()
2394            .flat_map(|model| {
2395                let tok = match load_tokenizer(model) {
2396                    Ok(t) => t,
2397                    Err(e) => return vec![format!("{model}: load error: {e}")],
2398                };
2399                probe_ids
2400                    .iter()
2401                    .filter_map(|&id| {
2402                        let token = tok.id_to_token(id)?;
2403                        let back = tok.token_to_id(token)?;
2404                        if back != id {
2405                            Some(format!("{model}: id {id} → {token:?} → {back}"))
2406                        } else {
2407                            None
2408                        }
2409                    })
2410                    .collect()
2411            })
2412            .collect();
2413        assert!(
2414            failures.is_empty(),
2415            "id↔token roundtrip failures:\n{}",
2416            failures.join("\n")
2417        );
2418    }
2419
2420    // ── DecodeStream ────────────────────────────────────────────────────────
2421
2422    const STREAM_MODEL: &str = "Qwen/Qwen3-0.6B";
2423
2424    fn stream_tok() -> Tokenizer {
2425        load_tokenizer(STREAM_MODEL).expect("failed to load tokenizer")
2426    }
2427
2428    fn stream_collect(tok: &Tokenizer, ids: &[u32], skip: bool) -> (String, usize) {
2429        let mut buf = Vec::new();
2430        let mut prefix = String::new();
2431        let mut prefix_index = 0usize;
2432        let mut out = String::new();
2433        for &id in ids {
2434            let chunk: Option<String> = super::decode_stream_step(
2435                tok,
2436                vec![id],
2437                skip,
2438                &mut buf,
2439                &mut prefix,
2440                &mut prefix_index,
2441            )
2442            .unwrap();
2443            if let Some(c) = chunk {
2444                out.push_str(&c);
2445            }
2446        }
2447        (out, buf.len())
2448    }
2449
2450    #[test]
2451    fn decode_stream_reconstructs_ascii() {
2452        let tok = stream_tok();
2453        let text = "Hello, world! This is a streaming decode test.";
2454        let ids = tok.encode_with_special_tokens(text, false).unwrap();
2455        let (decoded, _) = stream_collect(&tok, &ids, false);
2456        assert_eq!(decoded, text);
2457    }
2458
2459    #[test]
2460    fn decode_stream_reconstructs_unicode() {
2461        let tok = stream_tok();
2462        let text = "日本語テスト: こんにちは 🌍 — привет мир";
2463        let ids = tok.encode_with_special_tokens(text, false).unwrap();
2464        let (decoded, _) = stream_collect(&tok, &ids, false);
2465        assert_eq!(decoded, text);
2466    }
2467
2468    #[test]
2469    fn decode_stream_reconstructs_code() {
2470        let tok = stream_tok();
2471        let text = r#"fn main() { println!("hello"); }"#;
2472        let ids = tok.encode_with_special_tokens(text, false).unwrap();
2473        let (decoded, _) = stream_collect(&tok, &ids, false);
2474        assert_eq!(decoded, text);
2475    }
2476
2477    #[test]
2478    fn decode_stream_empty_ids_no_output() {
2479        let tok = stream_tok();
2480        let (decoded, buf_len) = stream_collect(&tok, &[], false);
2481        assert!(decoded.is_empty());
2482        assert_eq!(buf_len, 0);
2483    }
2484
2485    #[test]
2486    fn decode_stream_single_token() {
2487        let tok = stream_tok();
2488        let ids = tok.encode_with_special_tokens("hello", false).unwrap();
2489        assert!(!ids.is_empty());
2490        let (decoded, _) = stream_collect(&tok, &ids[..1], false);
2491        assert!(!decoded.is_empty());
2492    }
2493
2494    #[test]
2495    fn decode_stream_batch_step_matches_sequential() {
2496        let tok = stream_tok();
2497        let text = "The quick brown fox jumps over the lazy dog.";
2498        let ids = tok.encode_with_special_tokens(text, false).unwrap();
2499        let (sequential, _) = stream_collect(&tok, &ids, false);
2500        let mut buf = Vec::new();
2501        let mut prefix = String::new();
2502        let mut prefix_index = 0usize;
2503        let batch: String = super::decode_stream_step(
2504            &tok,
2505            ids.clone(),
2506            false,
2507            &mut buf,
2508            &mut prefix,
2509            &mut prefix_index,
2510        )
2511        .unwrap()
2512        .unwrap_or_default();
2513        assert_eq!(sequential, batch);
2514    }
2515
2516    #[test]
2517    fn decode_stream_pre_seeded_only_returns_new_tokens() {
2518        let tok = stream_tok();
2519        let prompt = "The capital of France is";
2520        let cont = " Paris.";
2521        let prompt_ids = tok.encode_with_special_tokens(prompt, false).unwrap();
2522        let cont_ids = tok.encode_with_special_tokens(cont, false).unwrap();
2523        let mut buf = prompt_ids.clone();
2524        let mut prefix = String::new();
2525        let mut prefix_index = 0usize;
2526        let mut out = String::new();
2527        for &id in &cont_ids {
2528            let chunk: Option<String> = super::decode_stream_step(
2529                &tok,
2530                vec![id],
2531                false,
2532                &mut buf,
2533                &mut prefix,
2534                &mut prefix_index,
2535            )
2536            .unwrap();
2537            if let Some(c) = chunk {
2538                out.push_str(&c);
2539            }
2540        }
2541        assert_eq!(out, cont);
2542    }
2543
2544    #[test]
2545    fn decode_stream_skip_special_tokens() {
2546        let tok = load_tokenizer("mistralai/Mistral-Nemo-Instruct-2407").unwrap();
2547        let text = "hello";
2548        let ids_with = tok.encode_with_special_tokens(text, true).unwrap();
2549        let ids_without = tok.encode_with_special_tokens(text, false).unwrap();
2550        assert!(
2551            ids_with.len() > ids_without.len(),
2552            "expected BOS/EOS tokens"
2553        );
2554        let (with_sp, _) = stream_collect(&tok, &ids_with, false);
2555        let (no_sp, _) = stream_collect(&tok, &ids_with, true);
2556        assert_eq!(no_sp, text);
2557        assert!(with_sp.contains(&no_sp));
2558    }
2559
2560    #[test]
2561    fn decode_stream_buffer_does_not_grow_unboundedly() {
2562        let tok = stream_tok();
2563        let text = "word ".repeat(80);
2564        let ids = tok.encode_with_special_tokens(text.trim(), false).unwrap();
2565        let (_, final_buf_len) = stream_collect(&tok, &ids, false);
2566        assert!(
2567            final_buf_len < 10,
2568            "buffer grew to {final_buf_len} entries after {} tokens",
2569            ids.len()
2570        );
2571    }
2572
2573    #[test]
2574    fn decode_stream_chunks_are_non_empty_and_concatenate() {
2575        let tok = stream_tok();
2576        let text = "one two three four five six seven eight nine ten";
2577        let ids = tok.encode_with_special_tokens(text, false).unwrap();
2578        let mut buf = Vec::new();
2579        let mut prefix = String::new();
2580        let mut prefix_index = 0usize;
2581        let mut chunks: Vec<String> = Vec::new();
2582        for &id in &ids {
2583            let chunk: Option<String> = super::decode_stream_step(
2584                &tok,
2585                vec![id],
2586                false,
2587                &mut buf,
2588                &mut prefix,
2589                &mut prefix_index,
2590            )
2591            .unwrap();
2592            if let Some(c) = chunk {
2593                assert!(!c.is_empty(), "stream emitted an empty chunk");
2594                chunks.push(c);
2595            }
2596        }
2597        assert_eq!(chunks.concat(), text);
2598    }
2599
2600    /// Streaming decode silently skips unknown IDs instead of erroring, so
2601    /// a single OOV token (e.g. emitted in the gap between tokenizer vocab
2602    /// and embedding matrix on some Qwen FP8 checkpoints) doesn't kill the
2603    /// whole generation. Matches HuggingFace DecodeStream behavior.
2604    #[test]
2605    fn decode_stream_unknown_id_does_not_error() {
2606        let tok = stream_tok();
2607        let mut buf = Vec::new();
2608        let mut prefix = String::new();
2609        let mut prefix_index = 0usize;
2610        let result = super::decode_stream_step(
2611            &tok,
2612            vec![u32::MAX],
2613            false,
2614            &mut buf,
2615            &mut prefix,
2616            &mut prefix_index,
2617        );
2618        assert!(result.is_ok(), "expected Ok, got {result:?}");
2619    }
2620
2621    #[test]
2622    fn decode_stream_invalid_prefix_error_message() {
2623        let tok = stream_tok();
2624        let ids = tok.encode_with_special_tokens("hello", false).unwrap();
2625        let mut buf = ids.clone();
2626        let mut prefix = "ZZZZZZZ".to_string();
2627        let mut prefix_index = 0usize;
2628        let result: Result<Option<String>, String> = super::decode_stream_step(
2629            &tok,
2630            vec![*ids.last().unwrap()],
2631            false,
2632            &mut buf,
2633            &mut prefix,
2634            &mut prefix_index,
2635        );
2636        if let Err(msg) = result {
2637            assert!(
2638                msg.starts_with("Invalid prefix encountered"),
2639                "unexpected error: {msg:?}"
2640            );
2641        }
2642    }
2643}