Skip to main content

dynamo_tokenizers/
tiktoken.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::collections::HashSet;
5use std::path::Path;
6
7use base64::Engine as _;
8use rayon::prelude::*;
9use rustc_hash::FxHashMap;
10use tiktoken_rs::CoreBPE;
11
12use super::{
13    EncodeSegment, Encoding, Error, Result, TokenIdType,
14    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17/// Number of reserved special-token slots to generate when filling gaps in the vocabulary.
18/// Most tiktoken-based models reserve 256 IDs above the base vocabulary for special tokens.
19const DEFAULT_NUM_RESERVED_SPECIAL_TOKENS: u32 = 256;
20
21/// Kimi BPE pattern. `fastokens` takes its Kimi scanner fast path only for this exact string.
22const KIMI_PATTERN: &str = fastokens::tiktoken::KIMI_PATTERN;
23
24pub struct TikTokenTokenizer {
25    bpe: CoreBPE,
26    special_token_ids: HashSet<u32>,
27    special_tokens: Vec<String>,
28}
29
30pub(crate) fn sorted_special_token_strings(special_tokens: &FxHashMap<String, u32>) -> Vec<String> {
31    let mut strings: Vec<String> = special_tokens.keys().cloned().collect();
32    strings.sort();
33    strings.dedup();
34    strings
35}
36
37impl TikTokenTokenizer {
38    /// Create a TikTokenTokenizer from a tiktoken model file.
39    ///
40    /// # Arguments
41    /// * `path` - Path to the `.model` or `.tiktoken` file (base64 rank-per-line format)
42    /// * `pattern` - BPE regex pattern string
43    /// * `special_tokens` - Map of special token strings to their IDs
44    pub fn from_file(
45        path: &str,
46        pattern: &str,
47        special_tokens: FxHashMap<String, u32>,
48    ) -> Result<Self> {
49        let encoder = parse_tiktoken_file(path)?;
50        let special_token_ids: HashSet<u32> = special_tokens.values().copied().collect();
51        let special_token_strings = sorted_special_token_strings(&special_tokens);
52
53        let bpe = CoreBPE::new(encoder, special_tokens, pattern)
54            .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
55
56        Ok(Self {
57            bpe,
58            special_token_ids,
59            special_tokens: special_token_strings,
60        })
61    }
62
63    /// Create a TikTokenTokenizer from a tiktoken model file, auto-detecting
64    /// the BPE pattern from `config.json` and special tokens from `tokenizer_config.json`.
65    ///
66    /// The tiktoken file and config files must be in the same directory.
67    pub fn from_file_auto(path: &str) -> Result<Self> {
68        let file_path = Path::new(path);
69        let directory = file_path
70            .parent()
71            .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
72
73        let pattern = detect_bpe_pattern(directory)?;
74        let encoder = parse_tiktoken_file(path)?;
75        // Use max rank + 1 (not len) to avoid ID collisions with sparse/non-contiguous ranks
76        let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
77        let AddedTokens { tokens, skip_ids } = load_added_tokens(directory, num_base_tokens)?;
78        let special_token_strings = sorted_special_token_strings(&tokens);
79
80        let bpe = CoreBPE::new(encoder, tokens, pattern)
81            .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
82
83        Ok(Self {
84            bpe,
85            special_token_ids: skip_ids,
86            special_tokens: special_token_strings,
87        })
88    }
89
90    /// Atomic special-token strings registered with the underlying TikToken BPE.
91    /// Includes added tokens marked `"special": false`, unlike `special_token_ids`.
92    ///
93    /// [`CoreBPE::encode_with_special_tokens`] recognizes these strings outside ordinary
94    /// BPE. For non-overlapping token spellings, splitting immediately after a match
95    /// preserves tokenization.
96    pub fn special_tokens(&self) -> &[String] {
97        &self.special_tokens
98    }
99}
100
101impl Encoder for TikTokenTokenizer {
102    fn encode(&self, input: &str) -> Result<Encoding> {
103        let token_ids: Vec<u32> = self.bpe.encode_with_special_tokens(input);
104        Ok(Encoding::Sp(token_ids))
105    }
106
107    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
108        inputs.par_iter().map(|input| self.encode(input)).collect()
109    }
110
111    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
112        let mut token_ids = Vec::new();
113        for segment in segments {
114            let start = token_ids.len();
115            for_each_reference_piece(segment.text, |piece| {
116                if segment.allow_special {
117                    token_ids.extend(self.bpe.encode_with_special_tokens(piece));
118                } else {
119                    token_ids.extend(self.bpe.encode_ordinary(piece));
120                }
121            });
122            if !segment.allow_special {
123                crate::reject_control_ids_in_ordinary_segment(
124                    &token_ids[start..],
125                    &self.special_token_ids,
126                )?;
127            }
128        }
129        Ok(Encoding::Sp(token_ids))
130    }
131}
132
133/// Longest text, in characters, the reference Kimi tokenizer passes to tiktoken at once.
134const MAX_ENCODE_CHARS: usize = 400_000;
135
136/// Longest run of whitespace or non-whitespace characters the reference Kimi tokenizer
137/// encodes in one piece (<https://github.com/openai/tiktoken/issues/195>).
138const MAX_RUN_CHARS: usize = 25_000;
139
140/// Python's `str.isspace`: Unicode `White_Space` plus the ASCII information separators
141/// U+001C..U+001F, which Rust's [`char::is_whitespace`] does not include.
142fn is_python_space(c: char) -> bool {
143    c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
144}
145
146/// Calls `f`, in order, on the pieces the reference Kimi tokenizer
147/// (`TikTokenTokenizer._encode_text_piece` in the checkpoint's `tokenization_kimi.py`)
148/// encodes separately: `text` cut into chunks of [`MAX_ENCODE_CHARS`] characters, each
149/// split before any whitespace or non-whitespace run would exceed [`MAX_RUN_CHARS`]
150/// characters. BPE does not merge across these cuts, so encoding the text whole yields
151/// different ids whenever a cut applies.
152fn for_each_reference_piece<'a>(text: &'a str, mut f: impl FnMut(&'a str)) {
153    // At most MAX_RUN_CHARS bytes means at most that many characters: no cut applies.
154    if text.len() <= MAX_RUN_CHARS {
155        f(text);
156        return;
157    }
158    let mut chunk_start = 0;
159    let mut chunk_chars = 0;
160    for (index, _) in text.char_indices() {
161        if chunk_chars == MAX_ENCODE_CHARS {
162            split_long_runs(&text[chunk_start..index], &mut f);
163            chunk_start = index;
164            chunk_chars = 0;
165        }
166        chunk_chars += 1;
167    }
168    split_long_runs(&text[chunk_start..], &mut f);
169}
170
171/// Calls `f` on `chunk` split before any whitespace or non-whitespace run would exceed
172/// [`MAX_RUN_CHARS`] characters.
173fn split_long_runs<'a>(chunk: &'a str, f: &mut impl FnMut(&'a str)) {
174    let mut run_is_space = chunk.chars().next().is_some_and(is_python_space);
175    let mut run_len = 0;
176    let mut piece_start = 0;
177    for (index, c) in chunk.char_indices() {
178        let is_space = is_python_space(c);
179        if is_space != run_is_space {
180            run_is_space = is_space;
181            run_len = 1;
182        } else {
183            run_len += 1;
184            if run_len > MAX_RUN_CHARS {
185                f(&chunk[piece_start..index]);
186                piece_start = index;
187                run_len = 1;
188            }
189        }
190    }
191    f(&chunk[piece_start..]);
192}
193
194impl Decoder for TikTokenTokenizer {
195    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
196        let ids: Vec<u32> = if skip_special_tokens {
197            token_ids
198                .iter()
199                .filter(|&&id| !self.special_token_ids.contains(&id))
200                .copied()
201                .collect()
202        } else {
203            token_ids.to_vec()
204        };
205
206        // Try strict UTF-8 first: valid bytes get `Complete` with zero extra allocation
207        // (takes ownership of the Vec). This correctly handles vocabulary tokens whose
208        // raw bytes are EF BF BD (legitimate U+FFFD) -- they are valid UTF-8 and must
209        // not be confused with incomplete multi-byte sequences.
210        //
211        // On failure, fall back to lossy conversion so partial multi-byte sequences
212        // become U+FFFD, then classify via the trailing-FFFD heuristic. This path is
213        // only hit during incremental detokenization of byte-fallback tokens.
214        let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
215        match String::from_utf8(bytes) {
216            Ok(text) => Ok(DecodeResult::Complete(text)),
217            Err(e) => {
218                let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
219                Ok(DecodeResult::from_decoded(text))
220            }
221        }
222    }
223}
224
225impl Tokenizer for TikTokenTokenizer {
226    fn validate_prefix_cache(&self) -> Result<()> {
227        Ok(())
228    }
229
230    // `tiktoken_rs::CoreBPE`'s `encoder` field is `pub(crate)`, so this
231    // crate has no way to enumerate, size, or look up ids in the vocab.
232    // `token_to_id` is left at the trait default (unsupported) for the
233    // same reason.
234    fn vocab_size(&self) -> Option<usize> {
235        None
236    }
237
238    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
239        let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
240        ids.sort_unstable();
241        Ok(ids)
242    }
243
244    // tiktoken applies no post-processor: `encode`'s `add_special_tokens`
245    // option (handled entirely by this crate, not tiktoken_rs) never adds
246    // tokens beyond what the caller passed in.
247    fn num_special_tokens_added(&self) -> Result<usize> {
248        Ok(0)
249    }
250}
251
252/// Parse a tiktoken model file (base64-encoded token + rank per line).
253pub(crate) fn parse_tiktoken_file(path: &str) -> Result<FxHashMap<Vec<u8>, u32>> {
254    let contents = std::fs::read_to_string(path)
255        .map_err(|err| Error::msg(format!("Failed to read tiktoken file '{path}': {err}")))?;
256
257    let engine = base64::engine::general_purpose::STANDARD;
258    let mut encoder = FxHashMap::default();
259
260    for line in contents.lines() {
261        let line = line.trim();
262        if line.is_empty() {
263            continue;
264        }
265        let mut parts = line.split_whitespace();
266        let token_b64 = parts
267            .next()
268            .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no token): {line}")))?;
269        let rank_str = parts
270            .next()
271            .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no rank): {line}")))?;
272
273        let token_bytes = engine
274            .decode(token_b64)
275            .map_err(|err| Error::msg(format!("Invalid base64 in tiktoken file: {err}")))?;
276        let rank: u32 = rank_str
277            .parse()
278            .map_err(|err| Error::msg(format!("Invalid rank in tiktoken file: {err}")))?;
279
280        encoder.insert(token_bytes, rank);
281    }
282
283    Ok(encoder)
284}
285
286/// Detect the BPE pattern for a model by reading `model_type` from `config.json`.
287pub(crate) fn detect_bpe_pattern(directory: &Path) -> Result<&'static str> {
288    let model_type: String = crate::file_json_field(&directory.join("config.json"), "model_type")
289        .map_err(|err| {
290        Error::msg(format!("Failed to read model_type from config.json: {err}"))
291    })?;
292
293    match model_type.as_str() {
294        // baseten-admin/Kimi-2.5-text-nvfp4-v3 model has model_type: "deepseek_v3" in its config.json
295        // because Kimi K2.5 is built on the DeepSeek V3 architecture.
296        // it still ships the Kimi tiktoken tokenizer file, so the KIMI_PATTERN BPE regex is the
297        // correct pattern to use.  No pure DeepSeek V3 model uses tiktoken.model files
298        // (they use tokenizer.json instead) so this match is safe.
299        "kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "kimi_k31" | "kimi_linear"
300        | "deepseek_v3" => Ok(KIMI_PATTERN),
301        _ => Err(Error::msg(format!(
302            "Unsupported tiktoken model_type '{model_type}'. \
303             Currently supported: kimi, kimi_k2, kimi_k25, kimi_k3, kimi_k31, kimi_linear, deepseek_v3. \
304             To add a new model type, extend detect_bpe_pattern() in lib/tokenizers/src/tiktoken.rs \
305             with the appropriate BPE regex pattern. \
306             Alternatively, provide a tokenizer.json (HuggingFace format) instead."
307        ))),
308    }
309}
310
311/// Added tokens from `tokenizer_config.json`.
312pub(crate) struct AddedTokens {
313    /// Every added token; each encodes as one unit.
314    pub(crate) tokens: FxHashMap<String, u32>,
315    /// IDs `skip_special_tokens` removes: `"special": true` entries and reserved slots.
316    /// A missing `special` field counts as false, as in Hugging Face.
317    pub(crate) skip_ids: HashSet<u32>,
318}
319
320/// Load added tokens from `tokenizer_config.json` in the model directory.
321///
322/// Reads the `added_tokens_decoder` field which maps string token IDs to token definitions.
323/// Falls back to generating `<|reserved_token_{id}|>` names for unmapped IDs.
324pub(crate) fn load_added_tokens(directory: &Path, num_base_tokens: usize) -> Result<AddedTokens> {
325    let config_path = directory.join("tokenizer_config.json");
326    let mut tokens = FxHashMap::default();
327    let mut skip_ids = HashSet::new();
328
329    if !config_path.exists() {
330        // No tokenizer_config.json β€” generate default reserved tokens
331        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
332            let id = num_base_tokens as u32 + i;
333            tokens.insert(format!("<|reserved_token_{id}|>"), id);
334            skip_ids.insert(id);
335        }
336        return Ok(AddedTokens { tokens, skip_ids });
337    }
338
339    let contents = std::fs::read_to_string(&config_path)
340        .map_err(|err| Error::msg(format!("Failed to read tokenizer_config.json: {err}")))?;
341
342    let config: serde_json::Value = serde_json::from_str(&contents)
343        .map_err(|err| Error::msg(format!("Failed to parse tokenizer_config.json: {err}")))?;
344
345    if let Some(added_tokens) = config
346        .get("added_tokens_decoder")
347        .and_then(|v| v.as_object())
348    {
349        for (id_str, token_def) in added_tokens {
350            let id: u32 = id_str.parse().map_err(|err| {
351                Error::msg(format!(
352                    "Invalid token ID '{id_str}' in added_tokens_decoder: {err}"
353                ))
354            })?;
355
356            let content = token_def
357                .get("content")
358                .and_then(|v| v.as_str())
359                .unwrap_or_else(|| {
360                    // This shouldn't happen in well-formed configs, but handle gracefully
361                    tracing::warn!("Missing 'content' field for token ID {id}");
362                    ""
363                });
364
365            if !content.is_empty() {
366                tokens.insert(content.to_string(), id);
367                if token_def
368                    .get("special")
369                    .and_then(serde_json::Value::as_bool)
370                    == Some(true)
371                {
372                    skip_ids.insert(id);
373                }
374            }
375        }
376
377        // Fill in any gaps with reserved tokens for the expected range
378        let used_ids: HashSet<u32> = tokens.values().copied().collect();
379        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
380            let id = num_base_tokens as u32 + i;
381            if !used_ids.contains(&id) {
382                tokens.insert(format!("<|reserved_token_{id}|>"), id);
383                skip_ids.insert(id);
384            }
385        }
386    } else {
387        // No added_tokens_decoder β€” generate default reserved tokens
388        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
389            let id = num_base_tokens as u32 + i;
390            tokens.insert(format!("<|reserved_token_{id}|>"), id);
391            skip_ids.insert(id);
392        }
393    }
394
395    Ok(AddedTokens { tokens, skip_ids })
396}
397
398#[cfg(test)]
399mod tests {
400    use super::*;
401    use crate::DecodeStream;
402    use std::io::Write;
403    use std::sync::Arc;
404
405    fn create_test_tiktoken_file(dir: &Path) -> String {
406        let engine = base64::engine::general_purpose::STANDARD;
407        let mut content = String::new();
408
409        // Create some simple token entries: single bytes with sequential ranks
410        let tokens: Vec<(&[u8], u32)> = vec![
411            (b"h", 0),
412            (b"e", 1),
413            (b"l", 2),
414            (b"o", 3),
415            (b" ", 4),
416            (b"w", 5),
417            (b"r", 6),
418            (b"d", 7),
419            (b"he", 8),
420            (b"ll", 9),
421            (b"lo", 10),
422            (b"wo", 11),
423            (b"rl", 12),
424            (b"hel", 13),
425            (b"llo", 14),
426            (b"wor", 15),
427            (b"hell", 16),
428            (b"ello", 17),
429            (b"worl", 18),
430            (b"hello", 19),
431            (b"world", 20),
432        ];
433
434        for (token, rank) in tokens {
435            let encoded = engine.encode(token);
436            content.push_str(&format!("{encoded} {rank}\n"));
437        }
438
439        let file_path = dir.join("tiktoken.model");
440        let mut file = std::fs::File::create(&file_path).unwrap();
441        file.write_all(content.as_bytes()).unwrap();
442        file_path.to_str().unwrap().to_string()
443    }
444
445    fn create_test_config(dir: &Path, model_type: &str) {
446        let config = serde_json::json!({
447            "model_type": model_type,
448            "max_position_embeddings": 32768,
449            "eos_token_id": [21]
450        });
451        let file_path = dir.join("config.json");
452        std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
453    }
454
455    fn create_test_tokenizer_config(dir: &Path, num_base_tokens: usize) {
456        let mut added_tokens = serde_json::Map::new();
457        let bos_id = num_base_tokens;
458        let eos_id = num_base_tokens + 1;
459
460        added_tokens.insert(
461            bos_id.to_string(),
462            serde_json::json!({"content": "[BOS]", "special": true}),
463        );
464        added_tokens.insert(
465            eos_id.to_string(),
466            serde_json::json!({"content": "[EOS]", "special": true}),
467        );
468
469        let config = serde_json::json!({
470            "added_tokens_decoder": added_tokens
471        });
472
473        let file_path = dir.join("tokenizer_config.json");
474        std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
475    }
476
477    #[test]
478    fn test_parse_tiktoken_file() {
479        let dir = tempfile::tempdir().unwrap();
480        let file_path = create_test_tiktoken_file(dir.path());
481        let encoder = parse_tiktoken_file(&file_path).unwrap();
482        assert_eq!(encoder.len(), 21);
483        assert_eq!(encoder[b"hello".as_slice()], 19);
484        assert_eq!(encoder[b"world".as_slice()], 20);
485    }
486
487    #[test]
488    fn test_parse_tiktoken_file_missing() {
489        let result = parse_tiktoken_file("/nonexistent/path/tiktoken.model");
490        assert!(result.is_err());
491    }
492
493    #[test]
494    fn test_tiktoken_from_file() {
495        let dir = tempfile::tempdir().unwrap();
496        let file_path = create_test_tiktoken_file(dir.path());
497
498        let mut special_tokens = FxHashMap::default();
499        special_tokens.insert("[BOS]".to_string(), 21_u32);
500        special_tokens.insert("[EOS]".to_string(), 22_u32);
501
502        // Use a simple pattern for testing
503        let pattern = r"[\w]+|[^\w\s]+|\s+";
504
505        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
506
507        assert_eq!(
508            tokenizer.special_tokens(),
509            &["[BOS]".to_string(), "[EOS]".to_string()]
510        );
511
512        // Test encode
513        let encoding = tokenizer.encode("hello world").unwrap();
514        let ids = encoding.token_ids();
515        assert!(!ids.is_empty());
516
517        // Test decode roundtrip
518        let decoded: String = tokenizer.decode(ids, false).unwrap().into();
519        assert_eq!(decoded, "hello world");
520    }
521
522    #[test]
523    fn test_tiktoken_encoding_variant() {
524        let dir = tempfile::tempdir().unwrap();
525        let file_path = create_test_tiktoken_file(dir.path());
526
527        let special_tokens = FxHashMap::default();
528        let pattern = r"[\w]+|[^\w\s]+|\s+";
529
530        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
531        assert!(tokenizer.special_tokens().is_empty());
532        let encoding = tokenizer.encode("hello").unwrap();
533
534        // Verify it produces the Sp variant
535        match &encoding {
536            Encoding::Sp(_) => {}
537            other => panic!("Expected Encoding::Sp, got {:?}", other),
538        }
539    }
540
541    #[test]
542    fn test_tiktoken_skip_special_tokens() {
543        let dir = tempfile::tempdir().unwrap();
544        let file_path = create_test_tiktoken_file(dir.path());
545
546        let mut special_tokens = FxHashMap::default();
547        special_tokens.insert("[BOS]".to_string(), 21_u32);
548        special_tokens.insert("[EOS]".to_string(), 22_u32);
549
550        let pattern = r"[\w]+|[^\w\s]+|\s+";
551
552        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
553
554        // Encode hello and prepend/append special tokens
555        let encoding = tokenizer.encode("hello").unwrap();
556        let mut ids = vec![21u32]; // [BOS]
557        ids.extend(encoding.token_ids());
558        ids.push(22); // [EOS]
559
560        // Decode with skip_special_tokens=true should strip special tokens
561        let decoded_skip: String = tokenizer.decode(&ids, true).unwrap().into();
562        assert_eq!(decoded_skip, "hello");
563
564        // Decode with skip_special_tokens=false should include them
565        let decoded_all: String = tokenizer.decode(&ids, false).unwrap().into();
566        assert!(decoded_all.contains("hello"));
567    }
568
569    #[test]
570    fn test_special_token_ids_reports_configured_ids_sorted() {
571        let dir = tempfile::tempdir().unwrap();
572        let file_path = create_test_tiktoken_file(dir.path());
573
574        let mut special_tokens = FxHashMap::default();
575        special_tokens.insert("[EOS]".to_string(), 22_u32);
576        special_tokens.insert("[BOS]".to_string(), 21_u32);
577
578        let pattern = r"[\w]+|[^\w\s]+|\s+";
579        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
580
581        assert_eq!(tokenizer.special_token_ids().unwrap(), vec![21, 22]);
582    }
583
584    #[test]
585    fn test_vocab_size_and_token_to_id_are_unsupported() {
586        let dir = tempfile::tempdir().unwrap();
587        let file_path = create_test_tiktoken_file(dir.path());
588        let pattern = r"[\w]+|[^\w\s]+|\s+";
589        let tokenizer =
590            TikTokenTokenizer::from_file(&file_path, pattern, FxHashMap::default()).unwrap();
591
592        assert_eq!(tokenizer.vocab_size(), None);
593        assert!(tokenizer.token_to_id("hello").is_err());
594    }
595
596    #[test]
597    fn test_tiktoken_from_file_auto() {
598        let dir = tempfile::tempdir().unwrap();
599        let file_path = create_test_tiktoken_file(dir.path());
600
601        create_test_config(dir.path(), "kimi");
602        create_test_tokenizer_config(dir.path(), 21);
603
604        let mut expected_specials: Vec<String> = load_added_tokens(dir.path(), 21)
605            .unwrap()
606            .tokens
607            .into_keys()
608            .collect();
609        expected_specials.sort();
610        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
611        assert_eq!(tokenizer.special_tokens(), expected_specials);
612
613        // Basic encode/decode roundtrip
614        let encoding = tokenizer.encode("hello world").unwrap();
615        let ids = encoding.token_ids();
616        assert!(!ids.is_empty());
617
618        let decoded: String = tokenizer.decode(ids, false).unwrap().into();
619        assert_eq!(decoded, "hello world");
620
621        // K3.1 ships the same tiktoken vocabulary format under a new model type.
622        create_test_config(dir.path(), "kimi_k31");
623        let k31_tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
624        assert_eq!(
625            k31_tokenizer.encode("hello world").unwrap().token_ids(),
626            ids
627        );
628        let decoded: String = k31_tokenizer.decode(ids, false).unwrap().into();
629        assert_eq!(decoded, "hello world");
630    }
631
632    #[test]
633    fn test_detect_bpe_pattern_unknown() {
634        let dir = tempfile::tempdir().unwrap();
635        create_test_config(dir.path(), "unknown_model");
636        let result = detect_bpe_pattern(dir.path());
637        assert!(result.is_err());
638    }
639
640    #[test]
641    fn test_kimi_k3_uses_kimi_bpe_pattern() {
642        let dir = tempfile::tempdir().unwrap();
643        for model_type in ["kimi_k3", "kimi_k31"] {
644            create_test_config(dir.path(), model_type);
645            assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
646        }
647    }
648
649    #[test]
650    fn test_detect_bpe_pattern_kimi_linear() {
651        let dir = tempfile::tempdir().unwrap();
652        create_test_config(dir.path(), "kimi_linear");
653        assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
654    }
655
656    #[test]
657    fn test_load_special_tokens_no_config() {
658        let dir = tempfile::tempdir().unwrap();
659        let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
660        assert_eq!(tokens.len(), 256);
661        assert_eq!(tokens["<|reserved_token_100|>"], 100);
662        assert_eq!(tokens["<|reserved_token_355|>"], 355);
663    }
664
665    #[test]
666    fn test_load_special_tokens_with_config() {
667        let dir = tempfile::tempdir().unwrap();
668        create_test_tokenizer_config(dir.path(), 100);
669        let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
670        assert_eq!(tokens["[BOS]"], 100);
671        assert_eq!(tokens["[EOS]"], 101);
672        // Should also have reserved tokens filling gaps
673        assert!(tokens.len() > 2);
674    }
675
676    /// Kimi-style `added_tokens_decoder`: a control token marked special, a format marker
677    /// marked `"special": false`, and one with no `special` field (Hugging Face default false).
678    fn create_tokenizer_with_non_special_added_tokens(dir: &Path) -> TikTokenTokenizer {
679        let file_path = create_test_tiktoken_file(dir);
680        create_test_config(dir, "kimi_k3");
681        let config = serde_json::json!({"added_tokens_decoder": {
682            "21": {"content": "[BOS]", "special": true},
683            "22": {"content": "<|open|>", "special": false},
684            "23": {"content": "<|sep|>"},
685        }});
686        std::fs::write(dir.join("tokenizer_config.json"), config.to_string()).unwrap();
687        TikTokenTokenizer::from_file_auto(&file_path).unwrap()
688    }
689
690    #[test]
691    fn test_skip_special_tokens_keeps_non_special_added_tokens() {
692        let dir = tempfile::tempdir().unwrap();
693        let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
694
695        let encoding = tokenizer.encode("<|open|>hello").unwrap();
696        assert_eq!(encoding.token_ids(), &[22, 19]);
697
698        let special_ids = tokenizer.special_token_ids().unwrap();
699        assert!(special_ids.contains(&21));
700        assert!(special_ids.contains(&24), "reserved placeholder");
701        assert!(!special_ids.contains(&22));
702        assert!(!special_ids.contains(&23));
703
704        let ids = [21, 22, 19, 23];
705        let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
706        assert_eq!(skipped, "<|open|>hello<|sep|>");
707        let kept: String = tokenizer.decode(&ids, false).unwrap().into();
708        assert_eq!(kept, "[BOS]<|open|>hello<|sep|>");
709    }
710
711    #[test]
712    fn test_decode_stream_emits_leading_non_special_added_token() {
713        let dir = tempfile::tempdir().unwrap();
714        let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
715        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
716
717        let mut stream = DecodeStream::new(tokenizer_arc, &[19], true);
718        assert_eq!(stream.step(22).unwrap().as_deref(), Some("<|open|>"));
719        assert_eq!(stream.step(19).unwrap().as_deref(), Some("hello"));
720        assert_eq!(stream.step(21).unwrap(), None);
721        assert_eq!(stream.step(23).unwrap().as_deref(), Some("<|sep|>"));
722    }
723
724    /// Helper: create a tiktoken file that includes raw byte tokens (byte fallback tokens).
725    fn create_test_tiktoken_file_with_byte_tokens(dir: &Path) -> String {
726        let engine = base64::engine::general_purpose::STANDARD;
727        let mut content = String::new();
728
729        let tokens: Vec<(&[u8], u32)> = vec![
730            (b"h", 0),
731            (b"e", 1),
732            (b"l", 2),
733            (b"o", 3),
734            (b" ", 4),
735            (b"hello", 5),
736        ];
737
738        for (token, rank) in &tokens {
739            let encoded = engine.encode(token);
740            content.push_str(&format!("{encoded} {rank}\n"));
741        }
742
743        // Byte-fallback tokens: individual bytes that form CJK character "δ½ " (U+4F60)
744        // UTF-8 encoding: 0xE4 0xBD 0xA0
745        let byte_tokens: Vec<(Vec<u8>, u32)> =
746            vec![(vec![0xE4], 100), (vec![0xBD], 101), (vec![0xA0], 102)];
747
748        for (token, rank) in &byte_tokens {
749            let encoded = engine.encode(token);
750            content.push_str(&format!("{encoded} {rank}\n"));
751        }
752
753        // Bytes for emoji "πŸ˜€" (U+1F600) β€” 4-byte UTF-8: 0xF0 0x9F 0x98 0x80
754        let emoji_tokens: Vec<(Vec<u8>, u32)> = vec![
755            (vec![0xF0], 200),
756            (vec![0x9F], 201),
757            (vec![0x98], 202),
758            (vec![0x80], 203),
759        ];
760
761        for (token, rank) in &emoji_tokens {
762            let encoded = engine.encode(token);
763            content.push_str(&format!("{encoded} {rank}\n"));
764        }
765
766        // Legitimate U+FFFD token: valid UTF-8 bytes EF BF BD (replacement character
767        // as an actual vocabulary entry, not an artifact of lossy conversion)
768        let fffd_token: Vec<(Vec<u8>, u32)> = vec![(vec![0xEF, 0xBF, 0xBD], 300)];
769
770        for (token, rank) in &fffd_token {
771            let encoded = engine.encode(token);
772            content.push_str(&format!("{encoded} {rank}\n"));
773        }
774
775        let file_path = dir.join("tiktoken.model");
776        let mut file = std::fs::File::create(&file_path).unwrap();
777        file.write_all(content.as_bytes()).unwrap();
778        file_path.to_str().unwrap().to_string()
779    }
780
781    fn create_byte_token_tokenizer(dir: &Path) -> TikTokenTokenizer {
782        let file_path = create_test_tiktoken_file_with_byte_tokens(dir);
783        let special_tokens = FxHashMap::default();
784        let pattern = r"[\w]+|[^\w\s]+|\s+";
785        TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap()
786    }
787
788    /// Reproduces the original panic: decoding a single byte-fallback token that is
789    /// part of a multi-byte UTF-8 character. Before the fix, CoreBPE::decode() would
790    /// call String::from_utf8() on [0xE4] and error with "incomplete utf-8 byte sequence".
791    #[test]
792    fn test_decode_single_incomplete_utf8_byte_does_not_error() {
793        let dir = tempfile::tempdir().unwrap();
794        let tokenizer = create_byte_token_tokenizer(dir.path());
795
796        let result = tokenizer.decode(&[100], false);
797        assert!(
798            result.is_ok(),
799            "decode() should not error on incomplete UTF-8 bytes"
800        );
801        let decode_result = result.unwrap();
802        assert!(
803            decode_result.is_partial(),
804            "incomplete UTF-8 byte should produce DecodeResult::Partial, got: {:?}",
805            decode_result
806        );
807    }
808
809    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
810    #[test]
811    fn test_decode_two_of_three_utf8_bytes_does_not_error() {
812        let dir = tempfile::tempdir().unwrap();
813        let tokenizer = create_byte_token_tokenizer(dir.path());
814
815        let result = tokenizer.decode(&[100, 101], false);
816        assert!(result.is_ok());
817        let decode_result = result.unwrap();
818        assert!(
819            decode_result.is_partial(),
820            "incomplete 2-of-3 UTF-8 bytes should produce DecodeResult::Partial, got: {:?}",
821            decode_result
822        );
823    }
824
825    /// When all bytes of a multi-byte character are present, the concatenated bytes form
826    /// valid UTF-8, so this test passes both before and after the fix. It serves as a
827    /// correctness check that the lossy conversion doesn't corrupt complete characters.
828    #[test]
829    fn test_decode_complete_multibyte_utf8_produces_correct_char() {
830        let dir = tempfile::tempdir().unwrap();
831        let tokenizer = create_byte_token_tokenizer(dir.path());
832
833        let result = tokenizer.decode(&[100, 101, 102], false);
834        assert!(result.is_ok());
835        assert_eq!(String::from(result.unwrap()), "δ½ ");
836    }
837
838    /// All 4 emoji bytes together form valid UTF-8, so this passes both before and after
839    /// the fix. Validates that lossy conversion doesn't alter complete multi-byte sequences.
840    #[test]
841    fn test_decode_complete_4byte_emoji_from_byte_tokens() {
842        let dir = tempfile::tempdir().unwrap();
843        let tokenizer = create_byte_token_tokenizer(dir.path());
844
845        let result = tokenizer.decode(&[200, 201, 202, 203], false);
846        assert!(result.is_ok());
847        assert_eq!(String::from(result.unwrap()), "πŸ˜€");
848    }
849
850    /// Regression test: a vocabulary token whose raw bytes are EF BF BD (the valid
851    /// UTF-8 encoding of U+FFFD) must decode as `Complete`, not `Partial`. Before the
852    /// from_utf8 fast-path fix, from_utf8_lossy + the trailing-FFFD heuristic would
853    /// misclassify this as Partial, causing the incremental decoder to suppress it.
854    #[test]
855    fn test_decode_legitimate_replacement_char_token_is_complete() {
856        let dir = tempfile::tempdir().unwrap();
857        let tokenizer = create_byte_token_tokenizer(dir.path());
858
859        let result = tokenizer.decode(&[300], false);
860        assert!(result.is_ok());
861        let decode_result = result.unwrap();
862        assert!(
863            decode_result.is_complete(),
864            "legitimate U+FFFD vocab token must be Complete, got: {:?}",
865            decode_result
866        );
867        assert_eq!(decode_result.as_str(), "\u{FFFD}");
868    }
869
870    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
871    #[test]
872    fn test_decode_partial_emoji_does_not_error() {
873        let dir = tempfile::tempdir().unwrap();
874        let tokenizer = create_byte_token_tokenizer(dir.path());
875
876        let result = tokenizer.decode(&[200], false);
877        assert!(result.is_ok());
878        assert!(result.unwrap().is_partial());
879    }
880
881    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
882    #[test]
883    fn test_decode_mixed_ascii_and_incomplete_bytes() {
884        let dir = tempfile::tempdir().unwrap();
885        let tokenizer = create_byte_token_tokenizer(dir.path());
886
887        let result = tokenizer.decode(&[5, 100], false);
888        assert!(result.is_ok());
889        let decode_result = result.unwrap();
890        assert!(
891            decode_result.is_partial(),
892            "trailing incomplete byte should produce DecodeResult::Partial"
893        );
894        let text: String = decode_result.into();
895        assert!(
896            text.starts_with("hello"),
897            "should start with 'hello', got: {:?}",
898            text
899        );
900    }
901
902    /// End-to-end incremental detokenization: DecodeStream buffers partial bytes,
903    /// emits the complete character once all bytes arrive.
904    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
905    #[test]
906    fn test_decode_stream_incremental_multibyte_reassembly() {
907        let dir = tempfile::tempdir().unwrap();
908        let tokenizer = create_byte_token_tokenizer(dir.path());
909        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
910
911        let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
912
913        let r1 = stream.step(100).unwrap();
914        assert_eq!(r1, None, "first byte of 3-byte char should be buffered");
915
916        let r2 = stream.step(101).unwrap();
917        assert_eq!(r2, None, "second byte of 3-byte char should be buffered");
918
919        let r3 = stream.step(102).unwrap();
920        assert!(r3.is_some(), "third byte should complete the character");
921        assert_eq!(r3.unwrap(), "δ½ ");
922    }
923
924    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
925    #[test]
926    fn test_decode_stream_incremental_emoji_reassembly() {
927        let dir = tempfile::tempdir().unwrap();
928        let tokenizer = create_byte_token_tokenizer(dir.path());
929        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
930
931        let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
932
933        let r1 = stream.step(200).unwrap();
934        assert_eq!(r1, None, "byte 1/4 of emoji should be buffered");
935
936        let r2 = stream.step(201).unwrap();
937        assert_eq!(r2, None, "byte 2/4 of emoji should be buffered");
938
939        let r3 = stream.step(202).unwrap();
940        assert_eq!(r3, None, "byte 3/4 of emoji should be buffered");
941
942        let r4 = stream.step(203).unwrap();
943        assert!(r4.is_some(), "byte 4/4 should complete the emoji");
944        assert_eq!(r4.unwrap(), "πŸ˜€");
945    }
946
947    #[test]
948    fn test_tiktoken_encode_batch() {
949        let dir = tempfile::tempdir().unwrap();
950        let file_path = create_test_tiktoken_file(dir.path());
951
952        let special_tokens = FxHashMap::default();
953        let pattern = r"[\w]+|[^\w\s]+|\s+";
954
955        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
956
957        let inputs = &["hello", "world"];
958        let encodings = tokenizer.encode_batch(inputs).unwrap();
959        assert_eq!(encodings.len(), 2);
960
961        for (encoding, input) in encodings.iter().zip(inputs.iter()) {
962            let decoded: String = tokenizer
963                .decode(encoding.token_ids(), false)
964                .unwrap()
965                .into();
966            assert_eq!(decoded, *input);
967        }
968    }
969
970    /// Helper: create a tiktoken file containing all 256 single-byte tokens (ranks 0..255).
971    /// This gives a complete byte-level base vocabulary so any ASCII string can be encoded.
972    fn create_byte_level_tiktoken_file(dir: &Path) -> String {
973        let engine = base64::engine::general_purpose::STANDARD;
974        let mut content = String::new();
975        for byte_val in 0u16..256 {
976            let encoded = engine.encode([byte_val as u8]);
977            content.push_str(&format!("{encoded} {byte_val}\n"));
978        }
979        let file_path = dir.join("tiktoken.model");
980        std::fs::write(&file_path, &content).unwrap();
981        file_path.to_str().unwrap().to_string()
982    }
983
984    #[test]
985    fn test_segment_encoding_preserves_special_vs_ordinary_boundary() {
986        let dir = tempfile::tempdir().unwrap();
987        let file_path = create_byte_level_tiktoken_file(dir.path());
988        let marker = "<|open|>";
989        let marker_id = 256;
990        let mut special_tokens = FxHashMap::default();
991        special_tokens.insert(marker.to_string(), marker_id);
992        let tokenizer =
993            TikTokenTokenizer::from_file(&file_path, r"(?s).+", special_tokens).unwrap();
994
995        let structural = tokenizer
996            .encode_segments(&[EncodeSegment::control(marker)])
997            .unwrap();
998        let user_text = tokenizer
999            .encode_segments(&[EncodeSegment::ordinary(marker)])
1000            .unwrap();
1001
1002        assert_eq!(structural.token_ids(), &[marker_id]);
1003        assert_ne!(user_text.token_ids(), &[marker_id]);
1004        let decoded: String = tokenizer
1005            .decode(user_text.token_ids(), false)
1006            .unwrap()
1007            .into();
1008        assert_eq!(decoded, marker);
1009    }
1010
1011    /// Character lengths of [`for_each_reference_piece`]'s pieces, to compare with the
1012    /// Python reference.
1013    fn piece_chars(text: &str) -> Vec<usize> {
1014        let mut lengths = Vec::new();
1015        for_each_reference_piece(text, |piece| lengths.push(piece.chars().count()));
1016        lengths
1017    }
1018
1019    #[test]
1020    fn test_reference_pieces_match_kimi_reference_split() {
1021        // Expected lengths come from the checkpoint's `_encode_text_piece` split, run in
1022        // Python on the same strings.
1023        assert_eq!(piece_chars("hello world"), vec![11]);
1024        assert_eq!(
1025            piece_chars(&"a".repeat(60_000)),
1026            vec![25_000, 25_000, 10_000]
1027        );
1028        assert_eq!(
1029            piece_chars(&("a".repeat(30_000) + &" ".repeat(30_000))),
1030            vec![25_000, 30_000, 5_000]
1031        );
1032        assert_eq!(
1033            piece_chars(&("x".repeat(25_000) + " " + &"y".repeat(25_001))),
1034            vec![50_001, 1]
1035        );
1036        // Python's isspace counts U+001C..U+001F as whitespace, so the separators start a
1037        // new run; with `char::is_whitespace` this is one 30,000-character run, cut at
1038        // 25,000.
1039        assert_eq!(
1040            piece_chars(&("a".repeat(20_000) + &"\u{1c}".repeat(10_000))),
1041            vec![30_000]
1042        );
1043        // Lengths are in characters, not bytes.
1044        assert_eq!(piece_chars(&"Γ©".repeat(26_000)), vec![25_000, 1_000]);
1045        // Chunks of 400k characters, even without long runs.
1046        let chunked: String = "ab ".repeat(136_667).chars().take(410_000).collect();
1047        assert_eq!(piece_chars(&chunked), vec![400_000, 10_000]);
1048    }
1049
1050    #[test]
1051    fn test_segment_encoding_cuts_long_runs_like_the_reference() {
1052        // Byte vocabulary plus one merge ("aa"), so a cut inside an "aa" pair changes ids.
1053        let dir = tempfile::tempdir().unwrap();
1054        let engine = base64::engine::general_purpose::STANDARD;
1055        let mut content = String::new();
1056        for byte_val in 0u16..256 {
1057            let encoded = engine.encode([byte_val as u8]);
1058            content.push_str(&format!("{encoded} {byte_val}\n"));
1059        }
1060        content.push_str(&format!("{} 256\n", engine.encode(b"aa")));
1061        let file_path = dir.path().join("tiktoken.model");
1062        std::fs::write(&file_path, &content).unwrap();
1063        let tokenizer = TikTokenTokenizer::from_file(
1064            file_path.to_str().unwrap(),
1065            r"(?s).+",
1066            FxHashMap::default(),
1067        )
1068        .unwrap();
1069
1070        // One non-whitespace run of 25,001 characters: the reference encodes the first
1071        // 25,000 ("b" + 24,999 "a") and the last "a" separately.
1072        let text = "b".to_string() + &"a".repeat(25_000);
1073        let segmented = tokenizer
1074            .encode_segments(&[EncodeSegment::ordinary(&text)])
1075            .unwrap();
1076        let mut expected = tokenizer.bpe.encode_ordinary(&text[..25_000]);
1077        expected.extend(tokenizer.bpe.encode_ordinary(&text[25_000..]));
1078        assert_eq!(segmented.token_ids(), expected.as_slice());
1079        assert_ne!(
1080            segmented.token_ids(),
1081            tokenizer.bpe.encode_ordinary(&text).as_slice(),
1082            "the cut must change ids for this input, or the test proves nothing"
1083        );
1084    }
1085
1086    fn has_nontrivial_self_overlap(token: &str) -> bool {
1087        let bytes = token.as_bytes();
1088        (1..bytes.len()).any(|overlap| bytes[bytes.len() - overlap..] == bytes[..overlap])
1089    }
1090
1091    fn have_ambiguous_overlap(a: &str, b: &str) -> bool {
1092        let a = a.as_bytes();
1093        let b = b.as_bytes();
1094
1095        if a.windows(b.len()).any(|window| window == b)
1096            || b.windows(a.len()).any(|window| window == a)
1097        {
1098            return true;
1099        }
1100
1101        let max_overlap = a.len().min(b.len());
1102        (1..max_overlap).any(|overlap| {
1103            a[a.len() - overlap..] == b[..overlap] || b[b.len() - overlap..] == a[..overlap]
1104        })
1105    }
1106
1107    fn assert_special_split_invariant(
1108        tokenizer: &TikTokenTokenizer,
1109        left: &str,
1110        special: &str,
1111        right: &str,
1112    ) {
1113        let whole = tokenizer
1114            .encode(&format!("{left}{special}{right}"))
1115            .unwrap()
1116            .token_ids()
1117            .to_vec();
1118        let mut split = tokenizer
1119            .encode(&format!("{left}{special}"))
1120            .unwrap()
1121            .token_ids()
1122            .to_vec();
1123        split.extend_from_slice(tokenizer.encode(right).unwrap().token_ids());
1124        assert_eq!(
1125            whole, split,
1126            "splitting after registered special token {special:?} must preserve tokenization"
1127        );
1128    }
1129
1130    #[test]
1131    fn test_registered_special_tokens_are_cache_safe_boundaries() {
1132        let dir = tempfile::tempdir().unwrap();
1133        let file_path = create_byte_level_tiktoken_file(dir.path());
1134        create_test_config(dir.path(), "kimi");
1135        create_test_tokenizer_config(dir.path(), 256);
1136
1137        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1138        let specials = tokenizer.special_tokens();
1139        assert!(!specials.is_empty());
1140
1141        for (index, special) in specials.iter().enumerate() {
1142            assert!(
1143                !has_nontrivial_self_overlap(special),
1144                "registered special token has an ambiguous self-overlap: {special:?}"
1145            );
1146            for other in &specials[index + 1..] {
1147                assert!(
1148                    !have_ambiguous_overlap(special, other),
1149                    "registered special tokens overlap ambiguously: {special:?} and {other:?}"
1150                );
1151            }
1152
1153            for (left, right) in [
1154                ("ordinary prefix ", " ordinary suffix"),
1155                ("line before\n", "\tUnicode after: εŒ—δΊ¬ πŸ˜€"),
1156                ("", " trailing text"),
1157            ] {
1158                assert_special_split_invariant(&tokenizer, left, special, right);
1159            }
1160        }
1161
1162        let bos = specials
1163            .iter()
1164            .find(|token| token.as_str() == "[BOS]")
1165            .unwrap();
1166        let eos = specials
1167            .iter()
1168            .find(|token| token.as_str() == "[EOS]")
1169            .unwrap();
1170        assert_special_split_invariant(
1171            &tokenizer,
1172            "prefix ",
1173            bos,
1174            &format!("{eos}<|reserved_token_258|>Unicode 尾部"),
1175        );
1176    }
1177
1178    /// Regression test for Kimi K2.5 tokenizer inflation.
1179    ///
1180    /// Python's tokenization_kimi.py names unnamed reserved tokens by absolute ID:
1181    ///   `<|reserved_token_{absolute_id}|>`
1182    ///
1183    /// The Rust code previously used relative offsets (0..255) as the naming index,
1184    /// so when a prompt contained `<|reserved_token_258|>` the Rust tokenizer did NOT
1185    /// recognize it as a special token. Each occurrence was encoded as multiple BPE
1186    /// tokens instead of 1, inflating an 8192-token prompt to 9038 tokens and causing
1187    /// TRT-LLM to reject the request.
1188    #[test]
1189    fn test_reserved_token_absolute_id_naming_kimi_k25_regression() {
1190        let dir = tempfile::tempdir().unwrap();
1191        let file_path = create_byte_level_tiktoken_file(dir.path());
1192
1193        // config.json with kimi model type -> triggers KIMI_PATTERN
1194        create_test_config(dir.path(), "kimi");
1195
1196        // tokenizer_config.json: BOS at 256, EOS at 257. Base vocab is IDs 0..255.
1197        create_test_tokenizer_config(dir.path(), 256);
1198
1199        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1200
1201        // ID 256 = [BOS], ID 257 = [EOS].
1202        // ID 258 = first UNNAMED reserved token.
1203        //   With fix:   named <|reserved_token_258|>  (absolute ID)
1204        //   Before fix: named <|reserved_token_2|>    (relative offset)
1205
1206        // Single unnamed reserved token should be recognized as 1 special token.
1207        let single = "<|reserved_token_258|>";
1208        let enc = tokenizer.encode(single).unwrap();
1209        assert_eq!(
1210            enc.token_ids().len(),
1211            1,
1212            "'{single}' should be 1 special token, got {} tokens: {:?}. \
1213             This means fallback naming still uses relative offsets instead of absolute IDs.",
1214            enc.token_ids().len(),
1215            enc.token_ids()
1216        );
1217        assert_eq!(enc.token_ids()[0], 258);
1218
1219        // Multiple unnamed reserved tokens in sequence (mini version of the benchmark).
1220        // IDs 258..268 are all unnamed; with the fix they're <|reserved_token_258|>..267.
1221        let multi: String = (258u32..268)
1222            .map(|id| format!("<|reserved_token_{id}|>"))
1223            .collect();
1224        let enc_multi = tokenizer.encode(&multi).unwrap();
1225        assert_eq!(
1226            enc_multi.token_ids().len(),
1227            10,
1228            "10 reserved token strings should produce exactly 10 tokens, got {}: {:?}",
1229            enc_multi.token_ids().len(),
1230            enc_multi.token_ids()
1231        );
1232        let expected_ids: Vec<u32> = (258..268).collect();
1233        assert_eq!(enc_multi.token_ids(), &expected_ids);
1234    }
1235
1236    /// Confirm that the old relative-offset naming would cause token inflation.
1237    /// Manually builds a tokenizer whose special-token map uses the WRONG names
1238    /// (relative offsets), then shows the same string encodes as many tokens.
1239    #[test]
1240    fn test_relative_offset_naming_causes_inflation() {
1241        let dir = tempfile::tempdir().unwrap();
1242        let file_path = create_byte_level_tiktoken_file(dir.path());
1243
1244        let _encoder = parse_tiktoken_file(&file_path).unwrap();
1245        let num_base_tokens = 256usize;
1246
1247        // Build special tokens with the OLD (buggy) relative-offset naming
1248        let mut bad_special_tokens: FxHashMap<String, u32> = FxHashMap::default();
1249        bad_special_tokens.insert("[BOS]".to_string(), 256);
1250        bad_special_tokens.insert("[EOS]".to_string(), 257);
1251        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
1252            let id = num_base_tokens as u32 + i;
1253            if id != 256 && id != 257 {
1254                // OLD naming: relative offset i, not absolute id
1255                bad_special_tokens.insert(format!("<|reserved_token_{i}|>"), id);
1256            }
1257        }
1258
1259        let bad_tokenizer =
1260            TikTokenTokenizer::from_file(&file_path, KIMI_PATTERN, bad_special_tokens).unwrap();
1261
1262        // With the wrong naming, <|reserved_token_258|> is NOT recognized as special.
1263        // It gets split into byte-level BPE tokens -> many more than 1.
1264        let input = "<|reserved_token_258|>";
1265        let enc = bad_tokenizer.encode(input).unwrap();
1266        assert!(
1267            enc.token_ids().len() > 1,
1268            "With buggy relative-offset naming, '{}' should NOT be recognized as a \
1269             single special token. Got {} token(s): {:?}",
1270            input,
1271            enc.token_ids().len(),
1272            enc.token_ids()
1273        );
1274
1275        // Show the inflation: 10 reserved tokens produce far more than 10 BPE tokens.
1276        let multi: String = (258u32..268)
1277            .map(|id| format!("<|reserved_token_{id}|>"))
1278            .collect();
1279        let enc_multi = bad_tokenizer.encode(&multi).unwrap();
1280        assert!(
1281            enc_multi.token_ids().len() > 10,
1282            "With buggy naming, 10 reserved token strings should inflate beyond 10 tokens. \
1283             Got {}",
1284            enc_multi.token_ids().len(),
1285        );
1286    }
1287}