Skip to main content

dynamo_tokenizers/
tiktoken.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::collections::HashSet;
5use std::path::Path;
6
7use base64::Engine as _;
8use rayon::prelude::*;
9use rustc_hash::FxHashMap;
10use tiktoken_rs::CoreBPE;
11
12use super::{
13    EncodeSegment, Encoding, Error, Result, TokenIdType,
14    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17/// Number of reserved special-token slots to generate when filling gaps in the vocabulary.
18/// Most tiktoken-based models reserve 256 IDs above the base vocabulary for special tokens.
19const DEFAULT_NUM_RESERVED_SPECIAL_TOKENS: u32 = 256;
20
21/// Kimi BPE pattern. `fastokens` takes its Kimi scanner fast path only for this exact string.
22const KIMI_PATTERN: &str = fastokens::tiktoken::KIMI_PATTERN;
23
24pub struct TikTokenTokenizer {
25    bpe: CoreBPE,
26    special_token_ids: HashSet<u32>,
27    special_tokens: Vec<String>,
28}
29
30pub(crate) fn sorted_special_token_strings(special_tokens: &FxHashMap<String, u32>) -> Vec<String> {
31    let mut strings: Vec<String> = special_tokens.keys().cloned().collect();
32    strings.sort();
33    strings.dedup();
34    strings
35}
36
37impl TikTokenTokenizer {
38    /// Create a TikTokenTokenizer from a tiktoken model file.
39    ///
40    /// # Arguments
41    /// * `path` - Path to the `.model` or `.tiktoken` file (base64 rank-per-line format)
42    /// * `pattern` - BPE regex pattern string
43    /// * `special_tokens` - Map of special token strings to their IDs
44    pub fn from_file(
45        path: &str,
46        pattern: &str,
47        special_tokens: FxHashMap<String, u32>,
48    ) -> Result<Self> {
49        let encoder = parse_tiktoken_file(path)?;
50        let special_token_ids: HashSet<u32> = special_tokens.values().copied().collect();
51        let special_token_strings = sorted_special_token_strings(&special_tokens);
52
53        let bpe = CoreBPE::new(encoder, special_tokens, pattern)
54            .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
55
56        Ok(Self {
57            bpe,
58            special_token_ids,
59            special_tokens: special_token_strings,
60        })
61    }
62
63    /// Create a TikTokenTokenizer from a tiktoken model file, auto-detecting
64    /// the BPE pattern from `config.json` and special tokens from `tokenizer_config.json`.
65    ///
66    /// The tiktoken file and config files must be in the same directory.
67    pub fn from_file_auto(path: &str) -> Result<Self> {
68        let file_path = Path::new(path);
69        let directory = file_path
70            .parent()
71            .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
72
73        let pattern = detect_bpe_pattern(directory)?;
74        let encoder = parse_tiktoken_file(path)?;
75        // Use max rank + 1 (not len) to avoid ID collisions with sparse/non-contiguous ranks
76        let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
77        let AddedTokens { tokens, skip_ids } = load_added_tokens(directory, num_base_tokens)?;
78        let special_token_strings = sorted_special_token_strings(&tokens);
79
80        let bpe = CoreBPE::new(encoder, tokens, pattern)
81            .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
82
83        Ok(Self {
84            bpe,
85            special_token_ids: skip_ids,
86            special_tokens: special_token_strings,
87        })
88    }
89
90    /// Atomic special-token strings registered with the underlying TikToken BPE.
91    /// Includes added tokens marked `"special": false`, unlike `special_token_ids`.
92    ///
93    /// [`CoreBPE::encode_with_special_tokens`] recognizes these strings outside ordinary
94    /// BPE. For non-overlapping token spellings, splitting immediately after a match
95    /// preserves tokenization.
96    pub fn special_tokens(&self) -> &[String] {
97        &self.special_tokens
98    }
99}
100
101impl Encoder for TikTokenTokenizer {
102    fn encode(&self, input: &str) -> Result<Encoding> {
103        let token_ids: Vec<u32> = self.bpe.encode_with_special_tokens(input);
104        Ok(Encoding::Sp(token_ids))
105    }
106
107    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
108        inputs.par_iter().map(|input| self.encode(input)).collect()
109    }
110
111    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
112        let mut token_ids = Vec::new();
113        for segment in segments {
114            for_each_reference_piece(segment.text, |piece| {
115                if segment.allow_special {
116                    token_ids.extend(self.bpe.encode_with_special_tokens(piece));
117                } else {
118                    token_ids.extend(self.bpe.encode_ordinary(piece));
119                }
120            });
121        }
122        Ok(Encoding::Sp(token_ids))
123    }
124}
125
126/// Longest text, in characters, the reference Kimi tokenizer passes to tiktoken at once.
127const MAX_ENCODE_CHARS: usize = 400_000;
128
129/// Longest run of whitespace or non-whitespace characters the reference Kimi tokenizer
130/// encodes in one piece (<https://github.com/openai/tiktoken/issues/195>).
131const MAX_RUN_CHARS: usize = 25_000;
132
133/// Python's `str.isspace`: Unicode `White_Space` plus the ASCII information separators
134/// U+001C..U+001F, which Rust's [`char::is_whitespace`] does not include.
135fn is_python_space(c: char) -> bool {
136    c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
137}
138
139/// Calls `f`, in order, on the pieces the reference Kimi tokenizer
140/// (`TikTokenTokenizer._encode_text_piece` in the checkpoint's `tokenization_kimi.py`)
141/// encodes separately: `text` cut into chunks of [`MAX_ENCODE_CHARS`] characters, each
142/// split before any whitespace or non-whitespace run would exceed [`MAX_RUN_CHARS`]
143/// characters. BPE does not merge across these cuts, so encoding the text whole yields
144/// different ids whenever a cut applies.
145fn for_each_reference_piece<'a>(text: &'a str, mut f: impl FnMut(&'a str)) {
146    // At most MAX_RUN_CHARS bytes means at most that many characters: no cut applies.
147    if text.len() <= MAX_RUN_CHARS {
148        f(text);
149        return;
150    }
151    let mut chunk_start = 0;
152    let mut chunk_chars = 0;
153    for (index, _) in text.char_indices() {
154        if chunk_chars == MAX_ENCODE_CHARS {
155            split_long_runs(&text[chunk_start..index], &mut f);
156            chunk_start = index;
157            chunk_chars = 0;
158        }
159        chunk_chars += 1;
160    }
161    split_long_runs(&text[chunk_start..], &mut f);
162}
163
164/// Calls `f` on `chunk` split before any whitespace or non-whitespace run would exceed
165/// [`MAX_RUN_CHARS`] characters.
166fn split_long_runs<'a>(chunk: &'a str, f: &mut impl FnMut(&'a str)) {
167    let mut run_is_space = chunk.chars().next().is_some_and(is_python_space);
168    let mut run_len = 0;
169    let mut piece_start = 0;
170    for (index, c) in chunk.char_indices() {
171        let is_space = is_python_space(c);
172        if is_space != run_is_space {
173            run_is_space = is_space;
174            run_len = 1;
175        } else {
176            run_len += 1;
177            if run_len > MAX_RUN_CHARS {
178                f(&chunk[piece_start..index]);
179                piece_start = index;
180                run_len = 1;
181            }
182        }
183    }
184    f(&chunk[piece_start..]);
185}
186
187impl Decoder for TikTokenTokenizer {
188    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
189        let ids: Vec<u32> = if skip_special_tokens {
190            token_ids
191                .iter()
192                .filter(|&&id| !self.special_token_ids.contains(&id))
193                .copied()
194                .collect()
195        } else {
196            token_ids.to_vec()
197        };
198
199        // Try strict UTF-8 first: valid bytes get `Complete` with zero extra allocation
200        // (takes ownership of the Vec). This correctly handles vocabulary tokens whose
201        // raw bytes are EF BF BD (legitimate U+FFFD) -- they are valid UTF-8 and must
202        // not be confused with incomplete multi-byte sequences.
203        //
204        // On failure, fall back to lossy conversion so partial multi-byte sequences
205        // become U+FFFD, then classify via the trailing-FFFD heuristic. This path is
206        // only hit during incremental detokenization of byte-fallback tokens.
207        let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
208        match String::from_utf8(bytes) {
209            Ok(text) => Ok(DecodeResult::Complete(text)),
210            Err(e) => {
211                let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
212                Ok(DecodeResult::from_decoded(text))
213            }
214        }
215    }
216}
217
218impl Tokenizer for TikTokenTokenizer {
219    fn validate_prefix_cache(&self) -> Result<()> {
220        Ok(())
221    }
222
223    // `tiktoken_rs::CoreBPE`'s `encoder` field is `pub(crate)`, so this
224    // crate has no way to enumerate, size, or look up ids in the vocab.
225    // `token_to_id` is left at the trait default (unsupported) for the
226    // same reason.
227    fn vocab_size(&self) -> Option<usize> {
228        None
229    }
230
231    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
232        let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
233        ids.sort_unstable();
234        Ok(ids)
235    }
236
237    // tiktoken applies no post-processor: `encode`'s `add_special_tokens`
238    // option (handled entirely by this crate, not tiktoken_rs) never adds
239    // tokens beyond what the caller passed in.
240    fn num_special_tokens_added(&self) -> Result<usize> {
241        Ok(0)
242    }
243}
244
245/// Parse a tiktoken model file (base64-encoded token + rank per line).
246pub(crate) fn parse_tiktoken_file(path: &str) -> Result<FxHashMap<Vec<u8>, u32>> {
247    let contents = std::fs::read_to_string(path)
248        .map_err(|err| Error::msg(format!("Failed to read tiktoken file '{path}': {err}")))?;
249
250    let engine = base64::engine::general_purpose::STANDARD;
251    let mut encoder = FxHashMap::default();
252
253    for line in contents.lines() {
254        let line = line.trim();
255        if line.is_empty() {
256            continue;
257        }
258        let mut parts = line.split_whitespace();
259        let token_b64 = parts
260            .next()
261            .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no token): {line}")))?;
262        let rank_str = parts
263            .next()
264            .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no rank): {line}")))?;
265
266        let token_bytes = engine
267            .decode(token_b64)
268            .map_err(|err| Error::msg(format!("Invalid base64 in tiktoken file: {err}")))?;
269        let rank: u32 = rank_str
270            .parse()
271            .map_err(|err| Error::msg(format!("Invalid rank in tiktoken file: {err}")))?;
272
273        encoder.insert(token_bytes, rank);
274    }
275
276    Ok(encoder)
277}
278
279/// Detect the BPE pattern for a model by reading `model_type` from `config.json`.
280pub(crate) fn detect_bpe_pattern(directory: &Path) -> Result<&'static str> {
281    let model_type: String = crate::file_json_field(&directory.join("config.json"), "model_type")
282        .map_err(|err| {
283        Error::msg(format!("Failed to read model_type from config.json: {err}"))
284    })?;
285
286    match model_type.as_str() {
287        // baseten-admin/Kimi-2.5-text-nvfp4-v3 model has model_type: "deepseek_v3" in its config.json
288        // because Kimi K2.5 is built on the DeepSeek V3 architecture.
289        // it still ships the Kimi tiktoken tokenizer file, so the KIMI_PATTERN BPE regex is the
290        // correct pattern to use.  No pure DeepSeek V3 model uses tiktoken.model files
291        // (they use tokenizer.json instead) so this match is safe.
292        "kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "kimi_k31" | "kimi_linear"
293        | "deepseek_v3" => Ok(KIMI_PATTERN),
294        _ => Err(Error::msg(format!(
295            "Unsupported tiktoken model_type '{model_type}'. \
296             Currently supported: kimi, kimi_k2, kimi_k25, kimi_k3, kimi_k31, kimi_linear, deepseek_v3. \
297             To add a new model type, extend detect_bpe_pattern() in lib/tokenizers/src/tiktoken.rs \
298             with the appropriate BPE regex pattern. \
299             Alternatively, provide a tokenizer.json (HuggingFace format) instead."
300        ))),
301    }
302}
303
304/// Added tokens from `tokenizer_config.json`.
305pub(crate) struct AddedTokens {
306    /// Every added token; each encodes as one unit.
307    pub(crate) tokens: FxHashMap<String, u32>,
308    /// IDs `skip_special_tokens` removes: `"special": true` entries and reserved slots.
309    /// A missing `special` field counts as false, as in Hugging Face.
310    pub(crate) skip_ids: HashSet<u32>,
311}
312
313/// Load added tokens from `tokenizer_config.json` in the model directory.
314///
315/// Reads the `added_tokens_decoder` field which maps string token IDs to token definitions.
316/// Falls back to generating `<|reserved_token_{id}|>` names for unmapped IDs.
317pub(crate) fn load_added_tokens(directory: &Path, num_base_tokens: usize) -> Result<AddedTokens> {
318    let config_path = directory.join("tokenizer_config.json");
319    let mut tokens = FxHashMap::default();
320    let mut skip_ids = HashSet::new();
321
322    if !config_path.exists() {
323        // No tokenizer_config.json β€” generate default reserved tokens
324        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
325            let id = num_base_tokens as u32 + i;
326            tokens.insert(format!("<|reserved_token_{id}|>"), id);
327            skip_ids.insert(id);
328        }
329        return Ok(AddedTokens { tokens, skip_ids });
330    }
331
332    let contents = std::fs::read_to_string(&config_path)
333        .map_err(|err| Error::msg(format!("Failed to read tokenizer_config.json: {err}")))?;
334
335    let config: serde_json::Value = serde_json::from_str(&contents)
336        .map_err(|err| Error::msg(format!("Failed to parse tokenizer_config.json: {err}")))?;
337
338    if let Some(added_tokens) = config
339        .get("added_tokens_decoder")
340        .and_then(|v| v.as_object())
341    {
342        for (id_str, token_def) in added_tokens {
343            let id: u32 = id_str.parse().map_err(|err| {
344                Error::msg(format!(
345                    "Invalid token ID '{id_str}' in added_tokens_decoder: {err}"
346                ))
347            })?;
348
349            let content = token_def
350                .get("content")
351                .and_then(|v| v.as_str())
352                .unwrap_or_else(|| {
353                    // This shouldn't happen in well-formed configs, but handle gracefully
354                    tracing::warn!("Missing 'content' field for token ID {id}");
355                    ""
356                });
357
358            if !content.is_empty() {
359                tokens.insert(content.to_string(), id);
360                if token_def
361                    .get("special")
362                    .and_then(serde_json::Value::as_bool)
363                    == Some(true)
364                {
365                    skip_ids.insert(id);
366                }
367            }
368        }
369
370        // Fill in any gaps with reserved tokens for the expected range
371        let used_ids: HashSet<u32> = tokens.values().copied().collect();
372        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
373            let id = num_base_tokens as u32 + i;
374            if !used_ids.contains(&id) {
375                tokens.insert(format!("<|reserved_token_{id}|>"), id);
376                skip_ids.insert(id);
377            }
378        }
379    } else {
380        // No added_tokens_decoder β€” generate default reserved tokens
381        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
382            let id = num_base_tokens as u32 + i;
383            tokens.insert(format!("<|reserved_token_{id}|>"), id);
384            skip_ids.insert(id);
385        }
386    }
387
388    Ok(AddedTokens { tokens, skip_ids })
389}
390
391#[cfg(test)]
392mod tests {
393    use super::*;
394    use crate::DecodeStream;
395    use std::io::Write;
396    use std::sync::Arc;
397
398    fn create_test_tiktoken_file(dir: &Path) -> String {
399        let engine = base64::engine::general_purpose::STANDARD;
400        let mut content = String::new();
401
402        // Create some simple token entries: single bytes with sequential ranks
403        let tokens: Vec<(&[u8], u32)> = vec![
404            (b"h", 0),
405            (b"e", 1),
406            (b"l", 2),
407            (b"o", 3),
408            (b" ", 4),
409            (b"w", 5),
410            (b"r", 6),
411            (b"d", 7),
412            (b"he", 8),
413            (b"ll", 9),
414            (b"lo", 10),
415            (b"wo", 11),
416            (b"rl", 12),
417            (b"hel", 13),
418            (b"llo", 14),
419            (b"wor", 15),
420            (b"hell", 16),
421            (b"ello", 17),
422            (b"worl", 18),
423            (b"hello", 19),
424            (b"world", 20),
425        ];
426
427        for (token, rank) in tokens {
428            let encoded = engine.encode(token);
429            content.push_str(&format!("{encoded} {rank}\n"));
430        }
431
432        let file_path = dir.join("tiktoken.model");
433        let mut file = std::fs::File::create(&file_path).unwrap();
434        file.write_all(content.as_bytes()).unwrap();
435        file_path.to_str().unwrap().to_string()
436    }
437
438    fn create_test_config(dir: &Path, model_type: &str) {
439        let config = serde_json::json!({
440            "model_type": model_type,
441            "max_position_embeddings": 32768,
442            "eos_token_id": [21]
443        });
444        let file_path = dir.join("config.json");
445        std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
446    }
447
448    fn create_test_tokenizer_config(dir: &Path, num_base_tokens: usize) {
449        let mut added_tokens = serde_json::Map::new();
450        let bos_id = num_base_tokens;
451        let eos_id = num_base_tokens + 1;
452
453        added_tokens.insert(
454            bos_id.to_string(),
455            serde_json::json!({"content": "[BOS]", "special": true}),
456        );
457        added_tokens.insert(
458            eos_id.to_string(),
459            serde_json::json!({"content": "[EOS]", "special": true}),
460        );
461
462        let config = serde_json::json!({
463            "added_tokens_decoder": added_tokens
464        });
465
466        let file_path = dir.join("tokenizer_config.json");
467        std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
468    }
469
470    #[test]
471    fn test_parse_tiktoken_file() {
472        let dir = tempfile::tempdir().unwrap();
473        let file_path = create_test_tiktoken_file(dir.path());
474        let encoder = parse_tiktoken_file(&file_path).unwrap();
475        assert_eq!(encoder.len(), 21);
476        assert_eq!(encoder[b"hello".as_slice()], 19);
477        assert_eq!(encoder[b"world".as_slice()], 20);
478    }
479
480    #[test]
481    fn test_parse_tiktoken_file_missing() {
482        let result = parse_tiktoken_file("/nonexistent/path/tiktoken.model");
483        assert!(result.is_err());
484    }
485
486    #[test]
487    fn test_tiktoken_from_file() {
488        let dir = tempfile::tempdir().unwrap();
489        let file_path = create_test_tiktoken_file(dir.path());
490
491        let mut special_tokens = FxHashMap::default();
492        special_tokens.insert("[BOS]".to_string(), 21_u32);
493        special_tokens.insert("[EOS]".to_string(), 22_u32);
494
495        // Use a simple pattern for testing
496        let pattern = r"[\w]+|[^\w\s]+|\s+";
497
498        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
499
500        assert_eq!(
501            tokenizer.special_tokens(),
502            &["[BOS]".to_string(), "[EOS]".to_string()]
503        );
504
505        // Test encode
506        let encoding = tokenizer.encode("hello world").unwrap();
507        let ids = encoding.token_ids();
508        assert!(!ids.is_empty());
509
510        // Test decode roundtrip
511        let decoded: String = tokenizer.decode(ids, false).unwrap().into();
512        assert_eq!(decoded, "hello world");
513    }
514
515    #[test]
516    fn test_tiktoken_encoding_variant() {
517        let dir = tempfile::tempdir().unwrap();
518        let file_path = create_test_tiktoken_file(dir.path());
519
520        let special_tokens = FxHashMap::default();
521        let pattern = r"[\w]+|[^\w\s]+|\s+";
522
523        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
524        assert!(tokenizer.special_tokens().is_empty());
525        let encoding = tokenizer.encode("hello").unwrap();
526
527        // Verify it produces the Sp variant
528        match &encoding {
529            Encoding::Sp(_) => {}
530            other => panic!("Expected Encoding::Sp, got {:?}", other),
531        }
532    }
533
534    #[test]
535    fn test_tiktoken_skip_special_tokens() {
536        let dir = tempfile::tempdir().unwrap();
537        let file_path = create_test_tiktoken_file(dir.path());
538
539        let mut special_tokens = FxHashMap::default();
540        special_tokens.insert("[BOS]".to_string(), 21_u32);
541        special_tokens.insert("[EOS]".to_string(), 22_u32);
542
543        let pattern = r"[\w]+|[^\w\s]+|\s+";
544
545        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
546
547        // Encode hello and prepend/append special tokens
548        let encoding = tokenizer.encode("hello").unwrap();
549        let mut ids = vec![21u32]; // [BOS]
550        ids.extend(encoding.token_ids());
551        ids.push(22); // [EOS]
552
553        // Decode with skip_special_tokens=true should strip special tokens
554        let decoded_skip: String = tokenizer.decode(&ids, true).unwrap().into();
555        assert_eq!(decoded_skip, "hello");
556
557        // Decode with skip_special_tokens=false should include them
558        let decoded_all: String = tokenizer.decode(&ids, false).unwrap().into();
559        assert!(decoded_all.contains("hello"));
560    }
561
562    #[test]
563    fn test_special_token_ids_reports_configured_ids_sorted() {
564        let dir = tempfile::tempdir().unwrap();
565        let file_path = create_test_tiktoken_file(dir.path());
566
567        let mut special_tokens = FxHashMap::default();
568        special_tokens.insert("[EOS]".to_string(), 22_u32);
569        special_tokens.insert("[BOS]".to_string(), 21_u32);
570
571        let pattern = r"[\w]+|[^\w\s]+|\s+";
572        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
573
574        assert_eq!(tokenizer.special_token_ids().unwrap(), vec![21, 22]);
575    }
576
577    #[test]
578    fn test_vocab_size_and_token_to_id_are_unsupported() {
579        let dir = tempfile::tempdir().unwrap();
580        let file_path = create_test_tiktoken_file(dir.path());
581        let pattern = r"[\w]+|[^\w\s]+|\s+";
582        let tokenizer =
583            TikTokenTokenizer::from_file(&file_path, pattern, FxHashMap::default()).unwrap();
584
585        assert_eq!(tokenizer.vocab_size(), None);
586        assert!(tokenizer.token_to_id("hello").is_err());
587    }
588
589    #[test]
590    fn test_tiktoken_from_file_auto() {
591        let dir = tempfile::tempdir().unwrap();
592        let file_path = create_test_tiktoken_file(dir.path());
593
594        create_test_config(dir.path(), "kimi");
595        create_test_tokenizer_config(dir.path(), 21);
596
597        let mut expected_specials: Vec<String> = load_added_tokens(dir.path(), 21)
598            .unwrap()
599            .tokens
600            .into_keys()
601            .collect();
602        expected_specials.sort();
603        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
604        assert_eq!(tokenizer.special_tokens(), expected_specials);
605
606        // Basic encode/decode roundtrip
607        let encoding = tokenizer.encode("hello world").unwrap();
608        let ids = encoding.token_ids();
609        assert!(!ids.is_empty());
610
611        let decoded: String = tokenizer.decode(ids, false).unwrap().into();
612        assert_eq!(decoded, "hello world");
613
614        // K3.1 ships the same tiktoken vocabulary format under a new model type.
615        create_test_config(dir.path(), "kimi_k31");
616        let k31_tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
617        assert_eq!(
618            k31_tokenizer.encode("hello world").unwrap().token_ids(),
619            ids
620        );
621        let decoded: String = k31_tokenizer.decode(ids, false).unwrap().into();
622        assert_eq!(decoded, "hello world");
623    }
624
625    #[test]
626    fn test_detect_bpe_pattern_unknown() {
627        let dir = tempfile::tempdir().unwrap();
628        create_test_config(dir.path(), "unknown_model");
629        let result = detect_bpe_pattern(dir.path());
630        assert!(result.is_err());
631    }
632
633    #[test]
634    fn test_kimi_k3_uses_kimi_bpe_pattern() {
635        let dir = tempfile::tempdir().unwrap();
636        for model_type in ["kimi_k3", "kimi_k31"] {
637            create_test_config(dir.path(), model_type);
638            assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
639        }
640    }
641
642    #[test]
643    fn test_detect_bpe_pattern_kimi_linear() {
644        let dir = tempfile::tempdir().unwrap();
645        create_test_config(dir.path(), "kimi_linear");
646        assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
647    }
648
649    #[test]
650    fn test_load_special_tokens_no_config() {
651        let dir = tempfile::tempdir().unwrap();
652        let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
653        assert_eq!(tokens.len(), 256);
654        assert_eq!(tokens["<|reserved_token_100|>"], 100);
655        assert_eq!(tokens["<|reserved_token_355|>"], 355);
656    }
657
658    #[test]
659    fn test_load_special_tokens_with_config() {
660        let dir = tempfile::tempdir().unwrap();
661        create_test_tokenizer_config(dir.path(), 100);
662        let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
663        assert_eq!(tokens["[BOS]"], 100);
664        assert_eq!(tokens["[EOS]"], 101);
665        // Should also have reserved tokens filling gaps
666        assert!(tokens.len() > 2);
667    }
668
669    /// Kimi-style `added_tokens_decoder`: a control token marked special, a format marker
670    /// marked `"special": false`, and one with no `special` field (Hugging Face default false).
671    fn create_tokenizer_with_non_special_added_tokens(dir: &Path) -> TikTokenTokenizer {
672        let file_path = create_test_tiktoken_file(dir);
673        create_test_config(dir, "kimi_k3");
674        let config = serde_json::json!({"added_tokens_decoder": {
675            "21": {"content": "[BOS]", "special": true},
676            "22": {"content": "<|open|>", "special": false},
677            "23": {"content": "<|sep|>"},
678        }});
679        std::fs::write(dir.join("tokenizer_config.json"), config.to_string()).unwrap();
680        TikTokenTokenizer::from_file_auto(&file_path).unwrap()
681    }
682
683    #[test]
684    fn test_skip_special_tokens_keeps_non_special_added_tokens() {
685        let dir = tempfile::tempdir().unwrap();
686        let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
687
688        let encoding = tokenizer.encode("<|open|>hello").unwrap();
689        assert_eq!(encoding.token_ids(), &[22, 19]);
690
691        let special_ids = tokenizer.special_token_ids().unwrap();
692        assert!(special_ids.contains(&21));
693        assert!(special_ids.contains(&24), "reserved placeholder");
694        assert!(!special_ids.contains(&22));
695        assert!(!special_ids.contains(&23));
696
697        let ids = [21, 22, 19, 23];
698        let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
699        assert_eq!(skipped, "<|open|>hello<|sep|>");
700        let kept: String = tokenizer.decode(&ids, false).unwrap().into();
701        assert_eq!(kept, "[BOS]<|open|>hello<|sep|>");
702    }
703
704    #[test]
705    fn test_decode_stream_emits_leading_non_special_added_token() {
706        let dir = tempfile::tempdir().unwrap();
707        let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
708        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
709
710        let mut stream = DecodeStream::new(tokenizer_arc, &[19], true);
711        assert_eq!(stream.step(22).unwrap().as_deref(), Some("<|open|>"));
712        assert_eq!(stream.step(19).unwrap().as_deref(), Some("hello"));
713        assert_eq!(stream.step(21).unwrap(), None);
714        assert_eq!(stream.step(23).unwrap().as_deref(), Some("<|sep|>"));
715    }
716
717    /// Helper: create a tiktoken file that includes raw byte tokens (byte fallback tokens).
718    fn create_test_tiktoken_file_with_byte_tokens(dir: &Path) -> String {
719        let engine = base64::engine::general_purpose::STANDARD;
720        let mut content = String::new();
721
722        let tokens: Vec<(&[u8], u32)> = vec![
723            (b"h", 0),
724            (b"e", 1),
725            (b"l", 2),
726            (b"o", 3),
727            (b" ", 4),
728            (b"hello", 5),
729        ];
730
731        for (token, rank) in &tokens {
732            let encoded = engine.encode(token);
733            content.push_str(&format!("{encoded} {rank}\n"));
734        }
735
736        // Byte-fallback tokens: individual bytes that form CJK character "δ½ " (U+4F60)
737        // UTF-8 encoding: 0xE4 0xBD 0xA0
738        let byte_tokens: Vec<(Vec<u8>, u32)> =
739            vec![(vec![0xE4], 100), (vec![0xBD], 101), (vec![0xA0], 102)];
740
741        for (token, rank) in &byte_tokens {
742            let encoded = engine.encode(token);
743            content.push_str(&format!("{encoded} {rank}\n"));
744        }
745
746        // Bytes for emoji "πŸ˜€" (U+1F600) β€” 4-byte UTF-8: 0xF0 0x9F 0x98 0x80
747        let emoji_tokens: Vec<(Vec<u8>, u32)> = vec![
748            (vec![0xF0], 200),
749            (vec![0x9F], 201),
750            (vec![0x98], 202),
751            (vec![0x80], 203),
752        ];
753
754        for (token, rank) in &emoji_tokens {
755            let encoded = engine.encode(token);
756            content.push_str(&format!("{encoded} {rank}\n"));
757        }
758
759        // Legitimate U+FFFD token: valid UTF-8 bytes EF BF BD (replacement character
760        // as an actual vocabulary entry, not an artifact of lossy conversion)
761        let fffd_token: Vec<(Vec<u8>, u32)> = vec![(vec![0xEF, 0xBF, 0xBD], 300)];
762
763        for (token, rank) in &fffd_token {
764            let encoded = engine.encode(token);
765            content.push_str(&format!("{encoded} {rank}\n"));
766        }
767
768        let file_path = dir.join("tiktoken.model");
769        let mut file = std::fs::File::create(&file_path).unwrap();
770        file.write_all(content.as_bytes()).unwrap();
771        file_path.to_str().unwrap().to_string()
772    }
773
774    fn create_byte_token_tokenizer(dir: &Path) -> TikTokenTokenizer {
775        let file_path = create_test_tiktoken_file_with_byte_tokens(dir);
776        let special_tokens = FxHashMap::default();
777        let pattern = r"[\w]+|[^\w\s]+|\s+";
778        TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap()
779    }
780
781    /// Reproduces the original panic: decoding a single byte-fallback token that is
782    /// part of a multi-byte UTF-8 character. Before the fix, CoreBPE::decode() would
783    /// call String::from_utf8() on [0xE4] and error with "incomplete utf-8 byte sequence".
784    #[test]
785    fn test_decode_single_incomplete_utf8_byte_does_not_error() {
786        let dir = tempfile::tempdir().unwrap();
787        let tokenizer = create_byte_token_tokenizer(dir.path());
788
789        let result = tokenizer.decode(&[100], false);
790        assert!(
791            result.is_ok(),
792            "decode() should not error on incomplete UTF-8 bytes"
793        );
794        let decode_result = result.unwrap();
795        assert!(
796            decode_result.is_partial(),
797            "incomplete UTF-8 byte should produce DecodeResult::Partial, got: {:?}",
798            decode_result
799        );
800    }
801
802    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
803    #[test]
804    fn test_decode_two_of_three_utf8_bytes_does_not_error() {
805        let dir = tempfile::tempdir().unwrap();
806        let tokenizer = create_byte_token_tokenizer(dir.path());
807
808        let result = tokenizer.decode(&[100, 101], false);
809        assert!(result.is_ok());
810        let decode_result = result.unwrap();
811        assert!(
812            decode_result.is_partial(),
813            "incomplete 2-of-3 UTF-8 bytes should produce DecodeResult::Partial, got: {:?}",
814            decode_result
815        );
816    }
817
818    /// When all bytes of a multi-byte character are present, the concatenated bytes form
819    /// valid UTF-8, so this test passes both before and after the fix. It serves as a
820    /// correctness check that the lossy conversion doesn't corrupt complete characters.
821    #[test]
822    fn test_decode_complete_multibyte_utf8_produces_correct_char() {
823        let dir = tempfile::tempdir().unwrap();
824        let tokenizer = create_byte_token_tokenizer(dir.path());
825
826        let result = tokenizer.decode(&[100, 101, 102], false);
827        assert!(result.is_ok());
828        assert_eq!(String::from(result.unwrap()), "δ½ ");
829    }
830
831    /// All 4 emoji bytes together form valid UTF-8, so this passes both before and after
832    /// the fix. Validates that lossy conversion doesn't alter complete multi-byte sequences.
833    #[test]
834    fn test_decode_complete_4byte_emoji_from_byte_tokens() {
835        let dir = tempfile::tempdir().unwrap();
836        let tokenizer = create_byte_token_tokenizer(dir.path());
837
838        let result = tokenizer.decode(&[200, 201, 202, 203], false);
839        assert!(result.is_ok());
840        assert_eq!(String::from(result.unwrap()), "πŸ˜€");
841    }
842
843    /// Regression test: a vocabulary token whose raw bytes are EF BF BD (the valid
844    /// UTF-8 encoding of U+FFFD) must decode as `Complete`, not `Partial`. Before the
845    /// from_utf8 fast-path fix, from_utf8_lossy + the trailing-FFFD heuristic would
846    /// misclassify this as Partial, causing the incremental decoder to suppress it.
847    #[test]
848    fn test_decode_legitimate_replacement_char_token_is_complete() {
849        let dir = tempfile::tempdir().unwrap();
850        let tokenizer = create_byte_token_tokenizer(dir.path());
851
852        let result = tokenizer.decode(&[300], false);
853        assert!(result.is_ok());
854        let decode_result = result.unwrap();
855        assert!(
856            decode_result.is_complete(),
857            "legitimate U+FFFD vocab token must be Complete, got: {:?}",
858            decode_result
859        );
860        assert_eq!(decode_result.as_str(), "\u{FFFD}");
861    }
862
863    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
864    #[test]
865    fn test_decode_partial_emoji_does_not_error() {
866        let dir = tempfile::tempdir().unwrap();
867        let tokenizer = create_byte_token_tokenizer(dir.path());
868
869        let result = tokenizer.decode(&[200], false);
870        assert!(result.is_ok());
871        assert!(result.unwrap().is_partial());
872    }
873
874    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
875    #[test]
876    fn test_decode_mixed_ascii_and_incomplete_bytes() {
877        let dir = tempfile::tempdir().unwrap();
878        let tokenizer = create_byte_token_tokenizer(dir.path());
879
880        let result = tokenizer.decode(&[5, 100], false);
881        assert!(result.is_ok());
882        let decode_result = result.unwrap();
883        assert!(
884            decode_result.is_partial(),
885            "trailing incomplete byte should produce DecodeResult::Partial"
886        );
887        let text: String = decode_result.into();
888        assert!(
889            text.starts_with("hello"),
890            "should start with 'hello', got: {:?}",
891            text
892        );
893    }
894
895    /// End-to-end incremental detokenization: DecodeStream buffers partial bytes,
896    /// emits the complete character once all bytes arrive.
897    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
898    #[test]
899    fn test_decode_stream_incremental_multibyte_reassembly() {
900        let dir = tempfile::tempdir().unwrap();
901        let tokenizer = create_byte_token_tokenizer(dir.path());
902        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
903
904        let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
905
906        let r1 = stream.step(100).unwrap();
907        assert_eq!(r1, None, "first byte of 3-byte char should be buffered");
908
909        let r2 = stream.step(101).unwrap();
910        assert_eq!(r2, None, "second byte of 3-byte char should be buffered");
911
912        let r3 = stream.step(102).unwrap();
913        assert!(r3.is_some(), "third byte should complete the character");
914        assert_eq!(r3.unwrap(), "δ½ ");
915    }
916
917    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
918    #[test]
919    fn test_decode_stream_incremental_emoji_reassembly() {
920        let dir = tempfile::tempdir().unwrap();
921        let tokenizer = create_byte_token_tokenizer(dir.path());
922        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
923
924        let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
925
926        let r1 = stream.step(200).unwrap();
927        assert_eq!(r1, None, "byte 1/4 of emoji should be buffered");
928
929        let r2 = stream.step(201).unwrap();
930        assert_eq!(r2, None, "byte 2/4 of emoji should be buffered");
931
932        let r3 = stream.step(202).unwrap();
933        assert_eq!(r3, None, "byte 3/4 of emoji should be buffered");
934
935        let r4 = stream.step(203).unwrap();
936        assert!(r4.is_some(), "byte 4/4 should complete the emoji");
937        assert_eq!(r4.unwrap(), "πŸ˜€");
938    }
939
940    #[test]
941    fn test_tiktoken_encode_batch() {
942        let dir = tempfile::tempdir().unwrap();
943        let file_path = create_test_tiktoken_file(dir.path());
944
945        let special_tokens = FxHashMap::default();
946        let pattern = r"[\w]+|[^\w\s]+|\s+";
947
948        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
949
950        let inputs = &["hello", "world"];
951        let encodings = tokenizer.encode_batch(inputs).unwrap();
952        assert_eq!(encodings.len(), 2);
953
954        for (encoding, input) in encodings.iter().zip(inputs.iter()) {
955            let decoded: String = tokenizer
956                .decode(encoding.token_ids(), false)
957                .unwrap()
958                .into();
959            assert_eq!(decoded, *input);
960        }
961    }
962
963    /// Helper: create a tiktoken file containing all 256 single-byte tokens (ranks 0..255).
964    /// This gives a complete byte-level base vocabulary so any ASCII string can be encoded.
965    fn create_byte_level_tiktoken_file(dir: &Path) -> String {
966        let engine = base64::engine::general_purpose::STANDARD;
967        let mut content = String::new();
968        for byte_val in 0u16..256 {
969            let encoded = engine.encode([byte_val as u8]);
970            content.push_str(&format!("{encoded} {byte_val}\n"));
971        }
972        let file_path = dir.join("tiktoken.model");
973        std::fs::write(&file_path, &content).unwrap();
974        file_path.to_str().unwrap().to_string()
975    }
976
977    #[test]
978    fn test_segment_encoding_preserves_special_vs_ordinary_boundary() {
979        let dir = tempfile::tempdir().unwrap();
980        let file_path = create_byte_level_tiktoken_file(dir.path());
981        let marker = "<|open|>";
982        let marker_id = 256;
983        let mut special_tokens = FxHashMap::default();
984        special_tokens.insert(marker.to_string(), marker_id);
985        let tokenizer =
986            TikTokenTokenizer::from_file(&file_path, r"(?s).+", special_tokens).unwrap();
987
988        let structural = tokenizer
989            .encode_segments(&[EncodeSegment::control(marker)])
990            .unwrap();
991        let user_text = tokenizer
992            .encode_segments(&[EncodeSegment::ordinary(marker)])
993            .unwrap();
994
995        assert_eq!(structural.token_ids(), &[marker_id]);
996        assert_ne!(user_text.token_ids(), &[marker_id]);
997        let decoded: String = tokenizer
998            .decode(user_text.token_ids(), false)
999            .unwrap()
1000            .into();
1001        assert_eq!(decoded, marker);
1002    }
1003
1004    /// Character lengths of [`for_each_reference_piece`]'s pieces, to compare with the
1005    /// Python reference.
1006    fn piece_chars(text: &str) -> Vec<usize> {
1007        let mut lengths = Vec::new();
1008        for_each_reference_piece(text, |piece| lengths.push(piece.chars().count()));
1009        lengths
1010    }
1011
1012    #[test]
1013    fn test_reference_pieces_match_kimi_reference_split() {
1014        // Expected lengths come from the checkpoint's `_encode_text_piece` split, run in
1015        // Python on the same strings.
1016        assert_eq!(piece_chars("hello world"), vec![11]);
1017        assert_eq!(
1018            piece_chars(&"a".repeat(60_000)),
1019            vec![25_000, 25_000, 10_000]
1020        );
1021        assert_eq!(
1022            piece_chars(&("a".repeat(30_000) + &" ".repeat(30_000))),
1023            vec![25_000, 30_000, 5_000]
1024        );
1025        assert_eq!(
1026            piece_chars(&("x".repeat(25_000) + " " + &"y".repeat(25_001))),
1027            vec![50_001, 1]
1028        );
1029        // Python's isspace counts U+001C..U+001F as whitespace, so the separators start a
1030        // new run; with `char::is_whitespace` this is one 30,000-character run, cut at
1031        // 25,000.
1032        assert_eq!(
1033            piece_chars(&("a".repeat(20_000) + &"\u{1c}".repeat(10_000))),
1034            vec![30_000]
1035        );
1036        // Lengths are in characters, not bytes.
1037        assert_eq!(piece_chars(&"Γ©".repeat(26_000)), vec![25_000, 1_000]);
1038        // Chunks of 400k characters, even without long runs.
1039        let chunked: String = "ab ".repeat(136_667).chars().take(410_000).collect();
1040        assert_eq!(piece_chars(&chunked), vec![400_000, 10_000]);
1041    }
1042
1043    #[test]
1044    fn test_segment_encoding_cuts_long_runs_like_the_reference() {
1045        // Byte vocabulary plus one merge ("aa"), so a cut inside an "aa" pair changes ids.
1046        let dir = tempfile::tempdir().unwrap();
1047        let engine = base64::engine::general_purpose::STANDARD;
1048        let mut content = String::new();
1049        for byte_val in 0u16..256 {
1050            let encoded = engine.encode([byte_val as u8]);
1051            content.push_str(&format!("{encoded} {byte_val}\n"));
1052        }
1053        content.push_str(&format!("{} 256\n", engine.encode(b"aa")));
1054        let file_path = dir.path().join("tiktoken.model");
1055        std::fs::write(&file_path, &content).unwrap();
1056        let tokenizer = TikTokenTokenizer::from_file(
1057            file_path.to_str().unwrap(),
1058            r"(?s).+",
1059            FxHashMap::default(),
1060        )
1061        .unwrap();
1062
1063        // One non-whitespace run of 25,001 characters: the reference encodes the first
1064        // 25,000 ("b" + 24,999 "a") and the last "a" separately.
1065        let text = "b".to_string() + &"a".repeat(25_000);
1066        let segmented = tokenizer
1067            .encode_segments(&[EncodeSegment::ordinary(&text)])
1068            .unwrap();
1069        let mut expected = tokenizer.bpe.encode_ordinary(&text[..25_000]);
1070        expected.extend(tokenizer.bpe.encode_ordinary(&text[25_000..]));
1071        assert_eq!(segmented.token_ids(), expected.as_slice());
1072        assert_ne!(
1073            segmented.token_ids(),
1074            tokenizer.bpe.encode_ordinary(&text).as_slice(),
1075            "the cut must change ids for this input, or the test proves nothing"
1076        );
1077    }
1078
1079    fn has_nontrivial_self_overlap(token: &str) -> bool {
1080        let bytes = token.as_bytes();
1081        (1..bytes.len()).any(|overlap| bytes[bytes.len() - overlap..] == bytes[..overlap])
1082    }
1083
1084    fn have_ambiguous_overlap(a: &str, b: &str) -> bool {
1085        let a = a.as_bytes();
1086        let b = b.as_bytes();
1087
1088        if a.windows(b.len()).any(|window| window == b)
1089            || b.windows(a.len()).any(|window| window == a)
1090        {
1091            return true;
1092        }
1093
1094        let max_overlap = a.len().min(b.len());
1095        (1..max_overlap).any(|overlap| {
1096            a[a.len() - overlap..] == b[..overlap] || b[b.len() - overlap..] == a[..overlap]
1097        })
1098    }
1099
1100    fn assert_special_split_invariant(
1101        tokenizer: &TikTokenTokenizer,
1102        left: &str,
1103        special: &str,
1104        right: &str,
1105    ) {
1106        let whole = tokenizer
1107            .encode(&format!("{left}{special}{right}"))
1108            .unwrap()
1109            .token_ids()
1110            .to_vec();
1111        let mut split = tokenizer
1112            .encode(&format!("{left}{special}"))
1113            .unwrap()
1114            .token_ids()
1115            .to_vec();
1116        split.extend_from_slice(tokenizer.encode(right).unwrap().token_ids());
1117        assert_eq!(
1118            whole, split,
1119            "splitting after registered special token {special:?} must preserve tokenization"
1120        );
1121    }
1122
1123    #[test]
1124    fn test_registered_special_tokens_are_cache_safe_boundaries() {
1125        let dir = tempfile::tempdir().unwrap();
1126        let file_path = create_byte_level_tiktoken_file(dir.path());
1127        create_test_config(dir.path(), "kimi");
1128        create_test_tokenizer_config(dir.path(), 256);
1129
1130        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1131        let specials = tokenizer.special_tokens();
1132        assert!(!specials.is_empty());
1133
1134        for (index, special) in specials.iter().enumerate() {
1135            assert!(
1136                !has_nontrivial_self_overlap(special),
1137                "registered special token has an ambiguous self-overlap: {special:?}"
1138            );
1139            for other in &specials[index + 1..] {
1140                assert!(
1141                    !have_ambiguous_overlap(special, other),
1142                    "registered special tokens overlap ambiguously: {special:?} and {other:?}"
1143                );
1144            }
1145
1146            for (left, right) in [
1147                ("ordinary prefix ", " ordinary suffix"),
1148                ("line before\n", "\tUnicode after: εŒ—δΊ¬ πŸ˜€"),
1149                ("", " trailing text"),
1150            ] {
1151                assert_special_split_invariant(&tokenizer, left, special, right);
1152            }
1153        }
1154
1155        let bos = specials
1156            .iter()
1157            .find(|token| token.as_str() == "[BOS]")
1158            .unwrap();
1159        let eos = specials
1160            .iter()
1161            .find(|token| token.as_str() == "[EOS]")
1162            .unwrap();
1163        assert_special_split_invariant(
1164            &tokenizer,
1165            "prefix ",
1166            bos,
1167            &format!("{eos}<|reserved_token_258|>Unicode 尾部"),
1168        );
1169    }
1170
1171    /// Regression test for Kimi K2.5 tokenizer inflation.
1172    ///
1173    /// Python's tokenization_kimi.py names unnamed reserved tokens by absolute ID:
1174    ///   `<|reserved_token_{absolute_id}|>`
1175    ///
1176    /// The Rust code previously used relative offsets (0..255) as the naming index,
1177    /// so when a prompt contained `<|reserved_token_258|>` the Rust tokenizer did NOT
1178    /// recognize it as a special token. Each occurrence was encoded as multiple BPE
1179    /// tokens instead of 1, inflating an 8192-token prompt to 9038 tokens and causing
1180    /// TRT-LLM to reject the request.
1181    #[test]
1182    fn test_reserved_token_absolute_id_naming_kimi_k25_regression() {
1183        let dir = tempfile::tempdir().unwrap();
1184        let file_path = create_byte_level_tiktoken_file(dir.path());
1185
1186        // config.json with kimi model type -> triggers KIMI_PATTERN
1187        create_test_config(dir.path(), "kimi");
1188
1189        // tokenizer_config.json: BOS at 256, EOS at 257. Base vocab is IDs 0..255.
1190        create_test_tokenizer_config(dir.path(), 256);
1191
1192        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1193
1194        // ID 256 = [BOS], ID 257 = [EOS].
1195        // ID 258 = first UNNAMED reserved token.
1196        //   With fix:   named <|reserved_token_258|>  (absolute ID)
1197        //   Before fix: named <|reserved_token_2|>    (relative offset)
1198
1199        // Single unnamed reserved token should be recognized as 1 special token.
1200        let single = "<|reserved_token_258|>";
1201        let enc = tokenizer.encode(single).unwrap();
1202        assert_eq!(
1203            enc.token_ids().len(),
1204            1,
1205            "'{single}' should be 1 special token, got {} tokens: {:?}. \
1206             This means fallback naming still uses relative offsets instead of absolute IDs.",
1207            enc.token_ids().len(),
1208            enc.token_ids()
1209        );
1210        assert_eq!(enc.token_ids()[0], 258);
1211
1212        // Multiple unnamed reserved tokens in sequence (mini version of the benchmark).
1213        // IDs 258..268 are all unnamed; with the fix they're <|reserved_token_258|>..267.
1214        let multi: String = (258u32..268)
1215            .map(|id| format!("<|reserved_token_{id}|>"))
1216            .collect();
1217        let enc_multi = tokenizer.encode(&multi).unwrap();
1218        assert_eq!(
1219            enc_multi.token_ids().len(),
1220            10,
1221            "10 reserved token strings should produce exactly 10 tokens, got {}: {:?}",
1222            enc_multi.token_ids().len(),
1223            enc_multi.token_ids()
1224        );
1225        let expected_ids: Vec<u32> = (258..268).collect();
1226        assert_eq!(enc_multi.token_ids(), &expected_ids);
1227    }
1228
1229    /// Confirm that the old relative-offset naming would cause token inflation.
1230    /// Manually builds a tokenizer whose special-token map uses the WRONG names
1231    /// (relative offsets), then shows the same string encodes as many tokens.
1232    #[test]
1233    fn test_relative_offset_naming_causes_inflation() {
1234        let dir = tempfile::tempdir().unwrap();
1235        let file_path = create_byte_level_tiktoken_file(dir.path());
1236
1237        let _encoder = parse_tiktoken_file(&file_path).unwrap();
1238        let num_base_tokens = 256usize;
1239
1240        // Build special tokens with the OLD (buggy) relative-offset naming
1241        let mut bad_special_tokens: FxHashMap<String, u32> = FxHashMap::default();
1242        bad_special_tokens.insert("[BOS]".to_string(), 256);
1243        bad_special_tokens.insert("[EOS]".to_string(), 257);
1244        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
1245            let id = num_base_tokens as u32 + i;
1246            if id != 256 && id != 257 {
1247                // OLD naming: relative offset i, not absolute id
1248                bad_special_tokens.insert(format!("<|reserved_token_{i}|>"), id);
1249            }
1250        }
1251
1252        let bad_tokenizer =
1253            TikTokenTokenizer::from_file(&file_path, KIMI_PATTERN, bad_special_tokens).unwrap();
1254
1255        // With the wrong naming, <|reserved_token_258|> is NOT recognized as special.
1256        // It gets split into byte-level BPE tokens -> many more than 1.
1257        let input = "<|reserved_token_258|>";
1258        let enc = bad_tokenizer.encode(input).unwrap();
1259        assert!(
1260            enc.token_ids().len() > 1,
1261            "With buggy relative-offset naming, '{}' should NOT be recognized as a \
1262             single special token. Got {} token(s): {:?}",
1263            input,
1264            enc.token_ids().len(),
1265            enc.token_ids()
1266        );
1267
1268        // Show the inflation: 10 reserved tokens produce far more than 10 BPE tokens.
1269        let multi: String = (258u32..268)
1270            .map(|id| format!("<|reserved_token_{id}|>"))
1271            .collect();
1272        let enc_multi = bad_tokenizer.encode(&multi).unwrap();
1273        assert!(
1274            enc_multi.token_ids().len() > 10,
1275            "With buggy naming, 10 reserved token strings should inflate beyond 10 tokens. \
1276             Got {}",
1277            enc_multi.token_ids().len(),
1278        );
1279    }
1280}