Skip to main content

dynamo_tokenizers/
tiktoken.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::collections::HashSet;
5use std::path::Path;
6
7use base64::Engine as _;
8use rayon::prelude::*;
9use rustc_hash::FxHashMap;
10use tiktoken_rs::CoreBPE;
11
12use super::{
13    EncodeSegment, Encoding, Error, Result, TokenIdType,
14    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17/// Number of reserved special-token slots to generate when filling gaps in the vocabulary.
18/// Most tiktoken-based models reserve 256 IDs above the base vocabulary for special tokens.
19const DEFAULT_NUM_RESERVED_SPECIAL_TOKENS: u32 = 256;
20
21/// Kimi BPE pattern. `fastokens` takes its Kimi scanner fast path only for this exact string.
22const KIMI_PATTERN: &str = fastokens::tiktoken::KIMI_PATTERN;
23
24pub struct TikTokenTokenizer {
25    bpe: CoreBPE,
26    special_token_ids: HashSet<u32>,
27    special_tokens: Vec<String>,
28}
29
30pub(crate) fn sorted_special_token_strings(special_tokens: &FxHashMap<String, u32>) -> Vec<String> {
31    let mut strings: Vec<String> = special_tokens.keys().cloned().collect();
32    strings.sort();
33    strings.dedup();
34    strings
35}
36
37impl TikTokenTokenizer {
38    /// Create a TikTokenTokenizer from a tiktoken model file.
39    ///
40    /// # Arguments
41    /// * `path` - Path to the `.model` or `.tiktoken` file (base64 rank-per-line format)
42    /// * `pattern` - BPE regex pattern string
43    /// * `special_tokens` - Map of special token strings to their IDs
44    pub fn from_file(
45        path: &str,
46        pattern: &str,
47        special_tokens: FxHashMap<String, u32>,
48    ) -> Result<Self> {
49        let encoder = parse_tiktoken_file(path)?;
50        let special_token_ids: HashSet<u32> = special_tokens.values().copied().collect();
51        let special_token_strings = sorted_special_token_strings(&special_tokens);
52
53        let bpe = CoreBPE::new(encoder, special_tokens, pattern)
54            .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
55
56        Ok(Self {
57            bpe,
58            special_token_ids,
59            special_tokens: special_token_strings,
60        })
61    }
62
63    /// Create a TikTokenTokenizer from a tiktoken model file, auto-detecting
64    /// the BPE pattern from `config.json` and special tokens from `tokenizer_config.json`.
65    ///
66    /// The tiktoken file and config files must be in the same directory.
67    pub fn from_file_auto(path: &str) -> Result<Self> {
68        let file_path = Path::new(path);
69        let directory = file_path
70            .parent()
71            .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
72
73        let pattern = detect_bpe_pattern(directory)?;
74        let encoder = parse_tiktoken_file(path)?;
75        // Use max rank + 1 (not len) to avoid ID collisions with sparse/non-contiguous ranks
76        let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
77        let AddedTokens { tokens, skip_ids } = load_added_tokens(directory, num_base_tokens)?;
78        let special_token_strings = sorted_special_token_strings(&tokens);
79
80        let bpe = CoreBPE::new(encoder, tokens, pattern)
81            .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
82
83        Ok(Self {
84            bpe,
85            special_token_ids: skip_ids,
86            special_tokens: special_token_strings,
87        })
88    }
89
90    /// Atomic special-token strings registered with the underlying TikToken BPE.
91    /// Includes added tokens marked `"special": false`, unlike `special_token_ids`.
92    ///
93    /// [`CoreBPE::encode_with_special_tokens`] recognizes these strings outside ordinary
94    /// BPE. For non-overlapping token spellings, splitting immediately after a match
95    /// preserves tokenization.
96    pub fn special_tokens(&self) -> &[String] {
97        &self.special_tokens
98    }
99}
100
101impl Encoder for TikTokenTokenizer {
102    fn encode(&self, input: &str) -> Result<Encoding> {
103        let token_ids: Vec<u32> = self.bpe.encode_with_special_tokens(input);
104        Ok(Encoding::Sp(token_ids))
105    }
106
107    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
108        inputs.par_iter().map(|input| self.encode(input)).collect()
109    }
110
111    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
112        let mut token_ids = Vec::new();
113        for segment in segments {
114            if segment.allow_special {
115                token_ids.extend(self.bpe.encode_with_special_tokens(segment.text));
116            } else {
117                token_ids.extend(self.bpe.encode_ordinary(segment.text));
118            }
119        }
120        Ok(Encoding::Sp(token_ids))
121    }
122}
123
124impl Decoder for TikTokenTokenizer {
125    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
126        let ids: Vec<u32> = if skip_special_tokens {
127            token_ids
128                .iter()
129                .filter(|&&id| !self.special_token_ids.contains(&id))
130                .copied()
131                .collect()
132        } else {
133            token_ids.to_vec()
134        };
135
136        // Try strict UTF-8 first: valid bytes get `Complete` with zero extra allocation
137        // (takes ownership of the Vec). This correctly handles vocabulary tokens whose
138        // raw bytes are EF BF BD (legitimate U+FFFD) -- they are valid UTF-8 and must
139        // not be confused with incomplete multi-byte sequences.
140        //
141        // On failure, fall back to lossy conversion so partial multi-byte sequences
142        // become U+FFFD, then classify via the trailing-FFFD heuristic. This path is
143        // only hit during incremental detokenization of byte-fallback tokens.
144        let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
145        match String::from_utf8(bytes) {
146            Ok(text) => Ok(DecodeResult::Complete(text)),
147            Err(e) => {
148                let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
149                Ok(DecodeResult::from_decoded(text))
150            }
151        }
152    }
153}
154
155impl Tokenizer for TikTokenTokenizer {
156    fn validate_prefix_cache(&self) -> Result<()> {
157        Ok(())
158    }
159
160    // `tiktoken_rs::CoreBPE`'s `encoder` field is `pub(crate)`, so this
161    // crate has no way to enumerate, size, or look up ids in the vocab.
162    // `token_to_id` is left at the trait default (unsupported) for the
163    // same reason.
164    fn vocab_size(&self) -> Option<usize> {
165        None
166    }
167
168    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
169        let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
170        ids.sort_unstable();
171        Ok(ids)
172    }
173
174    // tiktoken applies no post-processor: `encode`'s `add_special_tokens`
175    // option (handled entirely by this crate, not tiktoken_rs) never adds
176    // tokens beyond what the caller passed in.
177    fn num_special_tokens_added(&self) -> Result<usize> {
178        Ok(0)
179    }
180}
181
182/// Parse a tiktoken model file (base64-encoded token + rank per line).
183pub(crate) fn parse_tiktoken_file(path: &str) -> Result<FxHashMap<Vec<u8>, u32>> {
184    let contents = std::fs::read_to_string(path)
185        .map_err(|err| Error::msg(format!("Failed to read tiktoken file '{path}': {err}")))?;
186
187    let engine = base64::engine::general_purpose::STANDARD;
188    let mut encoder = FxHashMap::default();
189
190    for line in contents.lines() {
191        let line = line.trim();
192        if line.is_empty() {
193            continue;
194        }
195        let mut parts = line.split_whitespace();
196        let token_b64 = parts
197            .next()
198            .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no token): {line}")))?;
199        let rank_str = parts
200            .next()
201            .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no rank): {line}")))?;
202
203        let token_bytes = engine
204            .decode(token_b64)
205            .map_err(|err| Error::msg(format!("Invalid base64 in tiktoken file: {err}")))?;
206        let rank: u32 = rank_str
207            .parse()
208            .map_err(|err| Error::msg(format!("Invalid rank in tiktoken file: {err}")))?;
209
210        encoder.insert(token_bytes, rank);
211    }
212
213    Ok(encoder)
214}
215
216/// Detect the BPE pattern for a model by reading `model_type` from `config.json`.
217pub(crate) fn detect_bpe_pattern(directory: &Path) -> Result<&'static str> {
218    let model_type: String = crate::file_json_field(&directory.join("config.json"), "model_type")
219        .map_err(|err| {
220        Error::msg(format!("Failed to read model_type from config.json: {err}"))
221    })?;
222
223    match model_type.as_str() {
224        // baseten-admin/Kimi-2.5-text-nvfp4-v3 model has model_type: "deepseek_v3" in its config.json
225        // because Kimi K2.5 is built on the DeepSeek V3 architecture.
226        // it still ships the Kimi tiktoken tokenizer file, so the KIMI_PATTERN BPE regex is the
227        // correct pattern to use.  No pure DeepSeek V3 model uses tiktoken.model files
228        // (they use tokenizer.json instead) so this match is safe.
229        "kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "kimi_linear" | "deepseek_v3" => {
230            Ok(KIMI_PATTERN)
231        }
232        _ => Err(Error::msg(format!(
233            "Unsupported tiktoken model_type '{model_type}'. \
234             Currently supported: kimi, kimi_k2, kimi_k25, kimi_k3, kimi_linear, deepseek_v3. \
235             To add a new model type, extend detect_bpe_pattern() in lib/tokenizers/src/tiktoken.rs \
236             with the appropriate BPE regex pattern. \
237             Alternatively, provide a tokenizer.json (HuggingFace format) instead."
238        ))),
239    }
240}
241
242/// Added tokens from `tokenizer_config.json`.
243pub(crate) struct AddedTokens {
244    /// Every added token; each encodes as one unit.
245    pub(crate) tokens: FxHashMap<String, u32>,
246    /// IDs `skip_special_tokens` removes: `"special": true` entries and reserved slots.
247    /// A missing `special` field counts as false, as in Hugging Face.
248    pub(crate) skip_ids: HashSet<u32>,
249}
250
251/// Load added tokens from `tokenizer_config.json` in the model directory.
252///
253/// Reads the `added_tokens_decoder` field which maps string token IDs to token definitions.
254/// Falls back to generating `<|reserved_token_{id}|>` names for unmapped IDs.
255pub(crate) fn load_added_tokens(directory: &Path, num_base_tokens: usize) -> Result<AddedTokens> {
256    let config_path = directory.join("tokenizer_config.json");
257    let mut tokens = FxHashMap::default();
258    let mut skip_ids = HashSet::new();
259
260    if !config_path.exists() {
261        // No tokenizer_config.json β€” generate default reserved tokens
262        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
263            let id = num_base_tokens as u32 + i;
264            tokens.insert(format!("<|reserved_token_{id}|>"), id);
265            skip_ids.insert(id);
266        }
267        return Ok(AddedTokens { tokens, skip_ids });
268    }
269
270    let contents = std::fs::read_to_string(&config_path)
271        .map_err(|err| Error::msg(format!("Failed to read tokenizer_config.json: {err}")))?;
272
273    let config: serde_json::Value = serde_json::from_str(&contents)
274        .map_err(|err| Error::msg(format!("Failed to parse tokenizer_config.json: {err}")))?;
275
276    if let Some(added_tokens) = config
277        .get("added_tokens_decoder")
278        .and_then(|v| v.as_object())
279    {
280        for (id_str, token_def) in added_tokens {
281            let id: u32 = id_str.parse().map_err(|err| {
282                Error::msg(format!(
283                    "Invalid token ID '{id_str}' in added_tokens_decoder: {err}"
284                ))
285            })?;
286
287            let content = token_def
288                .get("content")
289                .and_then(|v| v.as_str())
290                .unwrap_or_else(|| {
291                    // This shouldn't happen in well-formed configs, but handle gracefully
292                    tracing::warn!("Missing 'content' field for token ID {id}");
293                    ""
294                });
295
296            if !content.is_empty() {
297                tokens.insert(content.to_string(), id);
298                if token_def
299                    .get("special")
300                    .and_then(serde_json::Value::as_bool)
301                    == Some(true)
302                {
303                    skip_ids.insert(id);
304                }
305            }
306        }
307
308        // Fill in any gaps with reserved tokens for the expected range
309        let used_ids: HashSet<u32> = tokens.values().copied().collect();
310        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
311            let id = num_base_tokens as u32 + i;
312            if !used_ids.contains(&id) {
313                tokens.insert(format!("<|reserved_token_{id}|>"), id);
314                skip_ids.insert(id);
315            }
316        }
317    } else {
318        // No added_tokens_decoder β€” generate default reserved tokens
319        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
320            let id = num_base_tokens as u32 + i;
321            tokens.insert(format!("<|reserved_token_{id}|>"), id);
322            skip_ids.insert(id);
323        }
324    }
325
326    Ok(AddedTokens { tokens, skip_ids })
327}
328
329#[cfg(test)]
330mod tests {
331    use super::*;
332    use crate::DecodeStream;
333    use std::io::Write;
334    use std::sync::Arc;
335
336    fn create_test_tiktoken_file(dir: &Path) -> String {
337        let engine = base64::engine::general_purpose::STANDARD;
338        let mut content = String::new();
339
340        // Create some simple token entries: single bytes with sequential ranks
341        let tokens: Vec<(&[u8], u32)> = vec![
342            (b"h", 0),
343            (b"e", 1),
344            (b"l", 2),
345            (b"o", 3),
346            (b" ", 4),
347            (b"w", 5),
348            (b"r", 6),
349            (b"d", 7),
350            (b"he", 8),
351            (b"ll", 9),
352            (b"lo", 10),
353            (b"wo", 11),
354            (b"rl", 12),
355            (b"hel", 13),
356            (b"llo", 14),
357            (b"wor", 15),
358            (b"hell", 16),
359            (b"ello", 17),
360            (b"worl", 18),
361            (b"hello", 19),
362            (b"world", 20),
363        ];
364
365        for (token, rank) in tokens {
366            let encoded = engine.encode(token);
367            content.push_str(&format!("{encoded} {rank}\n"));
368        }
369
370        let file_path = dir.join("tiktoken.model");
371        let mut file = std::fs::File::create(&file_path).unwrap();
372        file.write_all(content.as_bytes()).unwrap();
373        file_path.to_str().unwrap().to_string()
374    }
375
376    fn create_test_config(dir: &Path, model_type: &str) {
377        let config = serde_json::json!({
378            "model_type": model_type,
379            "max_position_embeddings": 32768,
380            "eos_token_id": [21]
381        });
382        let file_path = dir.join("config.json");
383        std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
384    }
385
386    fn create_test_tokenizer_config(dir: &Path, num_base_tokens: usize) {
387        let mut added_tokens = serde_json::Map::new();
388        let bos_id = num_base_tokens;
389        let eos_id = num_base_tokens + 1;
390
391        added_tokens.insert(
392            bos_id.to_string(),
393            serde_json::json!({"content": "[BOS]", "special": true}),
394        );
395        added_tokens.insert(
396            eos_id.to_string(),
397            serde_json::json!({"content": "[EOS]", "special": true}),
398        );
399
400        let config = serde_json::json!({
401            "added_tokens_decoder": added_tokens
402        });
403
404        let file_path = dir.join("tokenizer_config.json");
405        std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
406    }
407
408    #[test]
409    fn test_parse_tiktoken_file() {
410        let dir = tempfile::tempdir().unwrap();
411        let file_path = create_test_tiktoken_file(dir.path());
412        let encoder = parse_tiktoken_file(&file_path).unwrap();
413        assert_eq!(encoder.len(), 21);
414        assert_eq!(encoder[b"hello".as_slice()], 19);
415        assert_eq!(encoder[b"world".as_slice()], 20);
416    }
417
418    #[test]
419    fn test_parse_tiktoken_file_missing() {
420        let result = parse_tiktoken_file("/nonexistent/path/tiktoken.model");
421        assert!(result.is_err());
422    }
423
424    #[test]
425    fn test_tiktoken_from_file() {
426        let dir = tempfile::tempdir().unwrap();
427        let file_path = create_test_tiktoken_file(dir.path());
428
429        let mut special_tokens = FxHashMap::default();
430        special_tokens.insert("[BOS]".to_string(), 21_u32);
431        special_tokens.insert("[EOS]".to_string(), 22_u32);
432
433        // Use a simple pattern for testing
434        let pattern = r"[\w]+|[^\w\s]+|\s+";
435
436        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
437
438        assert_eq!(
439            tokenizer.special_tokens(),
440            &["[BOS]".to_string(), "[EOS]".to_string()]
441        );
442
443        // Test encode
444        let encoding = tokenizer.encode("hello world").unwrap();
445        let ids = encoding.token_ids();
446        assert!(!ids.is_empty());
447
448        // Test decode roundtrip
449        let decoded: String = tokenizer.decode(ids, false).unwrap().into();
450        assert_eq!(decoded, "hello world");
451    }
452
453    #[test]
454    fn test_tiktoken_encoding_variant() {
455        let dir = tempfile::tempdir().unwrap();
456        let file_path = create_test_tiktoken_file(dir.path());
457
458        let special_tokens = FxHashMap::default();
459        let pattern = r"[\w]+|[^\w\s]+|\s+";
460
461        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
462        assert!(tokenizer.special_tokens().is_empty());
463        let encoding = tokenizer.encode("hello").unwrap();
464
465        // Verify it produces the Sp variant
466        match &encoding {
467            Encoding::Sp(_) => {}
468            other => panic!("Expected Encoding::Sp, got {:?}", other),
469        }
470    }
471
472    #[test]
473    fn test_tiktoken_skip_special_tokens() {
474        let dir = tempfile::tempdir().unwrap();
475        let file_path = create_test_tiktoken_file(dir.path());
476
477        let mut special_tokens = FxHashMap::default();
478        special_tokens.insert("[BOS]".to_string(), 21_u32);
479        special_tokens.insert("[EOS]".to_string(), 22_u32);
480
481        let pattern = r"[\w]+|[^\w\s]+|\s+";
482
483        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
484
485        // Encode hello and prepend/append special tokens
486        let encoding = tokenizer.encode("hello").unwrap();
487        let mut ids = vec![21u32]; // [BOS]
488        ids.extend(encoding.token_ids());
489        ids.push(22); // [EOS]
490
491        // Decode with skip_special_tokens=true should strip special tokens
492        let decoded_skip: String = tokenizer.decode(&ids, true).unwrap().into();
493        assert_eq!(decoded_skip, "hello");
494
495        // Decode with skip_special_tokens=false should include them
496        let decoded_all: String = tokenizer.decode(&ids, false).unwrap().into();
497        assert!(decoded_all.contains("hello"));
498    }
499
500    #[test]
501    fn test_special_token_ids_reports_configured_ids_sorted() {
502        let dir = tempfile::tempdir().unwrap();
503        let file_path = create_test_tiktoken_file(dir.path());
504
505        let mut special_tokens = FxHashMap::default();
506        special_tokens.insert("[EOS]".to_string(), 22_u32);
507        special_tokens.insert("[BOS]".to_string(), 21_u32);
508
509        let pattern = r"[\w]+|[^\w\s]+|\s+";
510        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
511
512        assert_eq!(tokenizer.special_token_ids().unwrap(), vec![21, 22]);
513    }
514
515    #[test]
516    fn test_vocab_size_and_token_to_id_are_unsupported() {
517        let dir = tempfile::tempdir().unwrap();
518        let file_path = create_test_tiktoken_file(dir.path());
519        let pattern = r"[\w]+|[^\w\s]+|\s+";
520        let tokenizer =
521            TikTokenTokenizer::from_file(&file_path, pattern, FxHashMap::default()).unwrap();
522
523        assert_eq!(tokenizer.vocab_size(), None);
524        assert!(tokenizer.token_to_id("hello").is_err());
525    }
526
527    #[test]
528    fn test_tiktoken_from_file_auto() {
529        let dir = tempfile::tempdir().unwrap();
530        let file_path = create_test_tiktoken_file(dir.path());
531
532        create_test_config(dir.path(), "kimi");
533        create_test_tokenizer_config(dir.path(), 21);
534
535        let mut expected_specials: Vec<String> = load_added_tokens(dir.path(), 21)
536            .unwrap()
537            .tokens
538            .into_keys()
539            .collect();
540        expected_specials.sort();
541        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
542        assert_eq!(tokenizer.special_tokens(), expected_specials);
543
544        // Basic encode/decode roundtrip
545        let encoding = tokenizer.encode("hello world").unwrap();
546        let ids = encoding.token_ids();
547        assert!(!ids.is_empty());
548
549        let decoded: String = tokenizer.decode(ids, false).unwrap().into();
550        assert_eq!(decoded, "hello world");
551    }
552
553    #[test]
554    fn test_detect_bpe_pattern_unknown() {
555        let dir = tempfile::tempdir().unwrap();
556        create_test_config(dir.path(), "unknown_model");
557        let result = detect_bpe_pattern(dir.path());
558        assert!(result.is_err());
559    }
560
561    #[test]
562    fn test_kimi_k3_uses_kimi_bpe_pattern() {
563        let dir = tempfile::tempdir().unwrap();
564        create_test_config(dir.path(), "kimi_k3");
565        assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
566    }
567
568    #[test]
569    fn test_detect_bpe_pattern_kimi_linear() {
570        let dir = tempfile::tempdir().unwrap();
571        create_test_config(dir.path(), "kimi_linear");
572        assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
573    }
574
575    #[test]
576    fn test_load_special_tokens_no_config() {
577        let dir = tempfile::tempdir().unwrap();
578        let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
579        assert_eq!(tokens.len(), 256);
580        assert_eq!(tokens["<|reserved_token_100|>"], 100);
581        assert_eq!(tokens["<|reserved_token_355|>"], 355);
582    }
583
584    #[test]
585    fn test_load_special_tokens_with_config() {
586        let dir = tempfile::tempdir().unwrap();
587        create_test_tokenizer_config(dir.path(), 100);
588        let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
589        assert_eq!(tokens["[BOS]"], 100);
590        assert_eq!(tokens["[EOS]"], 101);
591        // Should also have reserved tokens filling gaps
592        assert!(tokens.len() > 2);
593    }
594
595    /// Kimi-style `added_tokens_decoder`: a control token marked special, a format marker
596    /// marked `"special": false`, and one with no `special` field (Hugging Face default false).
597    fn create_tokenizer_with_non_special_added_tokens(dir: &Path) -> TikTokenTokenizer {
598        let file_path = create_test_tiktoken_file(dir);
599        create_test_config(dir, "kimi_k3");
600        let config = serde_json::json!({"added_tokens_decoder": {
601            "21": {"content": "[BOS]", "special": true},
602            "22": {"content": "<|open|>", "special": false},
603            "23": {"content": "<|sep|>"},
604        }});
605        std::fs::write(dir.join("tokenizer_config.json"), config.to_string()).unwrap();
606        TikTokenTokenizer::from_file_auto(&file_path).unwrap()
607    }
608
609    #[test]
610    fn test_skip_special_tokens_keeps_non_special_added_tokens() {
611        let dir = tempfile::tempdir().unwrap();
612        let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
613
614        let encoding = tokenizer.encode("<|open|>hello").unwrap();
615        assert_eq!(encoding.token_ids(), &[22, 19]);
616
617        let special_ids = tokenizer.special_token_ids().unwrap();
618        assert!(special_ids.contains(&21));
619        assert!(special_ids.contains(&24), "reserved placeholder");
620        assert!(!special_ids.contains(&22));
621        assert!(!special_ids.contains(&23));
622
623        let ids = [21, 22, 19, 23];
624        let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
625        assert_eq!(skipped, "<|open|>hello<|sep|>");
626        let kept: String = tokenizer.decode(&ids, false).unwrap().into();
627        assert_eq!(kept, "[BOS]<|open|>hello<|sep|>");
628    }
629
630    #[test]
631    fn test_decode_stream_emits_leading_non_special_added_token() {
632        let dir = tempfile::tempdir().unwrap();
633        let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
634        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
635
636        let mut stream = DecodeStream::new(tokenizer_arc, &[19], true);
637        assert_eq!(stream.step(22).unwrap().as_deref(), Some("<|open|>"));
638        assert_eq!(stream.step(19).unwrap().as_deref(), Some("hello"));
639        assert_eq!(stream.step(21).unwrap(), None);
640        assert_eq!(stream.step(23).unwrap().as_deref(), Some("<|sep|>"));
641    }
642
643    /// Helper: create a tiktoken file that includes raw byte tokens (byte fallback tokens).
644    fn create_test_tiktoken_file_with_byte_tokens(dir: &Path) -> String {
645        let engine = base64::engine::general_purpose::STANDARD;
646        let mut content = String::new();
647
648        let tokens: Vec<(&[u8], u32)> = vec![
649            (b"h", 0),
650            (b"e", 1),
651            (b"l", 2),
652            (b"o", 3),
653            (b" ", 4),
654            (b"hello", 5),
655        ];
656
657        for (token, rank) in &tokens {
658            let encoded = engine.encode(token);
659            content.push_str(&format!("{encoded} {rank}\n"));
660        }
661
662        // Byte-fallback tokens: individual bytes that form CJK character "δ½ " (U+4F60)
663        // UTF-8 encoding: 0xE4 0xBD 0xA0
664        let byte_tokens: Vec<(Vec<u8>, u32)> =
665            vec![(vec![0xE4], 100), (vec![0xBD], 101), (vec![0xA0], 102)];
666
667        for (token, rank) in &byte_tokens {
668            let encoded = engine.encode(token);
669            content.push_str(&format!("{encoded} {rank}\n"));
670        }
671
672        // Bytes for emoji "πŸ˜€" (U+1F600) β€” 4-byte UTF-8: 0xF0 0x9F 0x98 0x80
673        let emoji_tokens: Vec<(Vec<u8>, u32)> = vec![
674            (vec![0xF0], 200),
675            (vec![0x9F], 201),
676            (vec![0x98], 202),
677            (vec![0x80], 203),
678        ];
679
680        for (token, rank) in &emoji_tokens {
681            let encoded = engine.encode(token);
682            content.push_str(&format!("{encoded} {rank}\n"));
683        }
684
685        // Legitimate U+FFFD token: valid UTF-8 bytes EF BF BD (replacement character
686        // as an actual vocabulary entry, not an artifact of lossy conversion)
687        let fffd_token: Vec<(Vec<u8>, u32)> = vec![(vec![0xEF, 0xBF, 0xBD], 300)];
688
689        for (token, rank) in &fffd_token {
690            let encoded = engine.encode(token);
691            content.push_str(&format!("{encoded} {rank}\n"));
692        }
693
694        let file_path = dir.join("tiktoken.model");
695        let mut file = std::fs::File::create(&file_path).unwrap();
696        file.write_all(content.as_bytes()).unwrap();
697        file_path.to_str().unwrap().to_string()
698    }
699
700    fn create_byte_token_tokenizer(dir: &Path) -> TikTokenTokenizer {
701        let file_path = create_test_tiktoken_file_with_byte_tokens(dir);
702        let special_tokens = FxHashMap::default();
703        let pattern = r"[\w]+|[^\w\s]+|\s+";
704        TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap()
705    }
706
707    /// Reproduces the original panic: decoding a single byte-fallback token that is
708    /// part of a multi-byte UTF-8 character. Before the fix, CoreBPE::decode() would
709    /// call String::from_utf8() on [0xE4] and error with "incomplete utf-8 byte sequence".
710    #[test]
711    fn test_decode_single_incomplete_utf8_byte_does_not_error() {
712        let dir = tempfile::tempdir().unwrap();
713        let tokenizer = create_byte_token_tokenizer(dir.path());
714
715        let result = tokenizer.decode(&[100], false);
716        assert!(
717            result.is_ok(),
718            "decode() should not error on incomplete UTF-8 bytes"
719        );
720        let decode_result = result.unwrap();
721        assert!(
722            decode_result.is_partial(),
723            "incomplete UTF-8 byte should produce DecodeResult::Partial, got: {:?}",
724            decode_result
725        );
726    }
727
728    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
729    #[test]
730    fn test_decode_two_of_three_utf8_bytes_does_not_error() {
731        let dir = tempfile::tempdir().unwrap();
732        let tokenizer = create_byte_token_tokenizer(dir.path());
733
734        let result = tokenizer.decode(&[100, 101], false);
735        assert!(result.is_ok());
736        let decode_result = result.unwrap();
737        assert!(
738            decode_result.is_partial(),
739            "incomplete 2-of-3 UTF-8 bytes should produce DecodeResult::Partial, got: {:?}",
740            decode_result
741        );
742    }
743
744    /// When all bytes of a multi-byte character are present, the concatenated bytes form
745    /// valid UTF-8, so this test passes both before and after the fix. It serves as a
746    /// correctness check that the lossy conversion doesn't corrupt complete characters.
747    #[test]
748    fn test_decode_complete_multibyte_utf8_produces_correct_char() {
749        let dir = tempfile::tempdir().unwrap();
750        let tokenizer = create_byte_token_tokenizer(dir.path());
751
752        let result = tokenizer.decode(&[100, 101, 102], false);
753        assert!(result.is_ok());
754        assert_eq!(String::from(result.unwrap()), "δ½ ");
755    }
756
757    /// All 4 emoji bytes together form valid UTF-8, so this passes both before and after
758    /// the fix. Validates that lossy conversion doesn't alter complete multi-byte sequences.
759    #[test]
760    fn test_decode_complete_4byte_emoji_from_byte_tokens() {
761        let dir = tempfile::tempdir().unwrap();
762        let tokenizer = create_byte_token_tokenizer(dir.path());
763
764        let result = tokenizer.decode(&[200, 201, 202, 203], false);
765        assert!(result.is_ok());
766        assert_eq!(String::from(result.unwrap()), "πŸ˜€");
767    }
768
769    /// Regression test: a vocabulary token whose raw bytes are EF BF BD (the valid
770    /// UTF-8 encoding of U+FFFD) must decode as `Complete`, not `Partial`. Before the
771    /// from_utf8 fast-path fix, from_utf8_lossy + the trailing-FFFD heuristic would
772    /// misclassify this as Partial, causing the incremental decoder to suppress it.
773    #[test]
774    fn test_decode_legitimate_replacement_char_token_is_complete() {
775        let dir = tempfile::tempdir().unwrap();
776        let tokenizer = create_byte_token_tokenizer(dir.path());
777
778        let result = tokenizer.decode(&[300], false);
779        assert!(result.is_ok());
780        let decode_result = result.unwrap();
781        assert!(
782            decode_result.is_complete(),
783            "legitimate U+FFFD vocab token must be Complete, got: {:?}",
784            decode_result
785        );
786        assert_eq!(decode_result.as_str(), "\u{FFFD}");
787    }
788
789    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
790    #[test]
791    fn test_decode_partial_emoji_does_not_error() {
792        let dir = tempfile::tempdir().unwrap();
793        let tokenizer = create_byte_token_tokenizer(dir.path());
794
795        let result = tokenizer.decode(&[200], false);
796        assert!(result.is_ok());
797        assert!(result.unwrap().is_partial());
798    }
799
800    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
801    #[test]
802    fn test_decode_mixed_ascii_and_incomplete_bytes() {
803        let dir = tempfile::tempdir().unwrap();
804        let tokenizer = create_byte_token_tokenizer(dir.path());
805
806        let result = tokenizer.decode(&[5, 100], false);
807        assert!(result.is_ok());
808        let decode_result = result.unwrap();
809        assert!(
810            decode_result.is_partial(),
811            "trailing incomplete byte should produce DecodeResult::Partial"
812        );
813        let text: String = decode_result.into();
814        assert!(
815            text.starts_with("hello"),
816            "should start with 'hello', got: {:?}",
817            text
818        );
819    }
820
821    /// End-to-end incremental detokenization: DecodeStream buffers partial bytes,
822    /// emits the complete character once all bytes arrive.
823    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
824    #[test]
825    fn test_decode_stream_incremental_multibyte_reassembly() {
826        let dir = tempfile::tempdir().unwrap();
827        let tokenizer = create_byte_token_tokenizer(dir.path());
828        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
829
830        let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
831
832        let r1 = stream.step(100).unwrap();
833        assert_eq!(r1, None, "first byte of 3-byte char should be buffered");
834
835        let r2 = stream.step(101).unwrap();
836        assert_eq!(r2, None, "second byte of 3-byte char should be buffered");
837
838        let r3 = stream.step(102).unwrap();
839        assert!(r3.is_some(), "third byte should complete the character");
840        assert_eq!(r3.unwrap(), "δ½ ");
841    }
842
843    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
844    #[test]
845    fn test_decode_stream_incremental_emoji_reassembly() {
846        let dir = tempfile::tempdir().unwrap();
847        let tokenizer = create_byte_token_tokenizer(dir.path());
848        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
849
850        let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
851
852        let r1 = stream.step(200).unwrap();
853        assert_eq!(r1, None, "byte 1/4 of emoji should be buffered");
854
855        let r2 = stream.step(201).unwrap();
856        assert_eq!(r2, None, "byte 2/4 of emoji should be buffered");
857
858        let r3 = stream.step(202).unwrap();
859        assert_eq!(r3, None, "byte 3/4 of emoji should be buffered");
860
861        let r4 = stream.step(203).unwrap();
862        assert!(r4.is_some(), "byte 4/4 should complete the emoji");
863        assert_eq!(r4.unwrap(), "πŸ˜€");
864    }
865
866    #[test]
867    fn test_tiktoken_encode_batch() {
868        let dir = tempfile::tempdir().unwrap();
869        let file_path = create_test_tiktoken_file(dir.path());
870
871        let special_tokens = FxHashMap::default();
872        let pattern = r"[\w]+|[^\w\s]+|\s+";
873
874        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
875
876        let inputs = &["hello", "world"];
877        let encodings = tokenizer.encode_batch(inputs).unwrap();
878        assert_eq!(encodings.len(), 2);
879
880        for (encoding, input) in encodings.iter().zip(inputs.iter()) {
881            let decoded: String = tokenizer
882                .decode(encoding.token_ids(), false)
883                .unwrap()
884                .into();
885            assert_eq!(decoded, *input);
886        }
887    }
888
889    /// Helper: create a tiktoken file containing all 256 single-byte tokens (ranks 0..255).
890    /// This gives a complete byte-level base vocabulary so any ASCII string can be encoded.
891    fn create_byte_level_tiktoken_file(dir: &Path) -> String {
892        let engine = base64::engine::general_purpose::STANDARD;
893        let mut content = String::new();
894        for byte_val in 0u16..256 {
895            let encoded = engine.encode([byte_val as u8]);
896            content.push_str(&format!("{encoded} {byte_val}\n"));
897        }
898        let file_path = dir.join("tiktoken.model");
899        std::fs::write(&file_path, &content).unwrap();
900        file_path.to_str().unwrap().to_string()
901    }
902
903    #[test]
904    fn test_segment_encoding_preserves_special_vs_ordinary_boundary() {
905        let dir = tempfile::tempdir().unwrap();
906        let file_path = create_byte_level_tiktoken_file(dir.path());
907        let marker = "<|open|>";
908        let marker_id = 256;
909        let mut special_tokens = FxHashMap::default();
910        special_tokens.insert(marker.to_string(), marker_id);
911        let tokenizer =
912            TikTokenTokenizer::from_file(&file_path, r"(?s).+", special_tokens).unwrap();
913
914        let structural = tokenizer
915            .encode_segments(&[EncodeSegment::control(marker)])
916            .unwrap();
917        let user_text = tokenizer
918            .encode_segments(&[EncodeSegment::ordinary(marker)])
919            .unwrap();
920
921        assert_eq!(structural.token_ids(), &[marker_id]);
922        assert_ne!(user_text.token_ids(), &[marker_id]);
923        let decoded: String = tokenizer
924            .decode(user_text.token_ids(), false)
925            .unwrap()
926            .into();
927        assert_eq!(decoded, marker);
928    }
929
930    fn has_nontrivial_self_overlap(token: &str) -> bool {
931        let bytes = token.as_bytes();
932        (1..bytes.len()).any(|overlap| bytes[bytes.len() - overlap..] == bytes[..overlap])
933    }
934
935    fn have_ambiguous_overlap(a: &str, b: &str) -> bool {
936        let a = a.as_bytes();
937        let b = b.as_bytes();
938
939        if a.windows(b.len()).any(|window| window == b)
940            || b.windows(a.len()).any(|window| window == a)
941        {
942            return true;
943        }
944
945        let max_overlap = a.len().min(b.len());
946        (1..max_overlap).any(|overlap| {
947            a[a.len() - overlap..] == b[..overlap] || b[b.len() - overlap..] == a[..overlap]
948        })
949    }
950
951    fn assert_special_split_invariant(
952        tokenizer: &TikTokenTokenizer,
953        left: &str,
954        special: &str,
955        right: &str,
956    ) {
957        let whole = tokenizer
958            .encode(&format!("{left}{special}{right}"))
959            .unwrap()
960            .token_ids()
961            .to_vec();
962        let mut split = tokenizer
963            .encode(&format!("{left}{special}"))
964            .unwrap()
965            .token_ids()
966            .to_vec();
967        split.extend_from_slice(tokenizer.encode(right).unwrap().token_ids());
968        assert_eq!(
969            whole, split,
970            "splitting after registered special token {special:?} must preserve tokenization"
971        );
972    }
973
974    #[test]
975    fn test_registered_special_tokens_are_cache_safe_boundaries() {
976        let dir = tempfile::tempdir().unwrap();
977        let file_path = create_byte_level_tiktoken_file(dir.path());
978        create_test_config(dir.path(), "kimi");
979        create_test_tokenizer_config(dir.path(), 256);
980
981        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
982        let specials = tokenizer.special_tokens();
983        assert!(!specials.is_empty());
984
985        for (index, special) in specials.iter().enumerate() {
986            assert!(
987                !has_nontrivial_self_overlap(special),
988                "registered special token has an ambiguous self-overlap: {special:?}"
989            );
990            for other in &specials[index + 1..] {
991                assert!(
992                    !have_ambiguous_overlap(special, other),
993                    "registered special tokens overlap ambiguously: {special:?} and {other:?}"
994                );
995            }
996
997            for (left, right) in [
998                ("ordinary prefix ", " ordinary suffix"),
999                ("line before\n", "\tUnicode after: εŒ—δΊ¬ πŸ˜€"),
1000                ("", " trailing text"),
1001            ] {
1002                assert_special_split_invariant(&tokenizer, left, special, right);
1003            }
1004        }
1005
1006        let bos = specials
1007            .iter()
1008            .find(|token| token.as_str() == "[BOS]")
1009            .unwrap();
1010        let eos = specials
1011            .iter()
1012            .find(|token| token.as_str() == "[EOS]")
1013            .unwrap();
1014        assert_special_split_invariant(
1015            &tokenizer,
1016            "prefix ",
1017            bos,
1018            &format!("{eos}<|reserved_token_258|>Unicode 尾部"),
1019        );
1020    }
1021
1022    /// Regression test for Kimi K2.5 tokenizer inflation.
1023    ///
1024    /// Python's tokenization_kimi.py names unnamed reserved tokens by absolute ID:
1025    ///   `<|reserved_token_{absolute_id}|>`
1026    ///
1027    /// The Rust code previously used relative offsets (0..255) as the naming index,
1028    /// so when a prompt contained `<|reserved_token_258|>` the Rust tokenizer did NOT
1029    /// recognize it as a special token. Each occurrence was encoded as multiple BPE
1030    /// tokens instead of 1, inflating an 8192-token prompt to 9038 tokens and causing
1031    /// TRT-LLM to reject the request.
1032    #[test]
1033    fn test_reserved_token_absolute_id_naming_kimi_k25_regression() {
1034        let dir = tempfile::tempdir().unwrap();
1035        let file_path = create_byte_level_tiktoken_file(dir.path());
1036
1037        // config.json with kimi model type -> triggers KIMI_PATTERN
1038        create_test_config(dir.path(), "kimi");
1039
1040        // tokenizer_config.json: BOS at 256, EOS at 257. Base vocab is IDs 0..255.
1041        create_test_tokenizer_config(dir.path(), 256);
1042
1043        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1044
1045        // ID 256 = [BOS], ID 257 = [EOS].
1046        // ID 258 = first UNNAMED reserved token.
1047        //   With fix:   named <|reserved_token_258|>  (absolute ID)
1048        //   Before fix: named <|reserved_token_2|>    (relative offset)
1049
1050        // Single unnamed reserved token should be recognized as 1 special token.
1051        let single = "<|reserved_token_258|>";
1052        let enc = tokenizer.encode(single).unwrap();
1053        assert_eq!(
1054            enc.token_ids().len(),
1055            1,
1056            "'{single}' should be 1 special token, got {} tokens: {:?}. \
1057             This means fallback naming still uses relative offsets instead of absolute IDs.",
1058            enc.token_ids().len(),
1059            enc.token_ids()
1060        );
1061        assert_eq!(enc.token_ids()[0], 258);
1062
1063        // Multiple unnamed reserved tokens in sequence (mini version of the benchmark).
1064        // IDs 258..268 are all unnamed; with the fix they're <|reserved_token_258|>..267.
1065        let multi: String = (258u32..268)
1066            .map(|id| format!("<|reserved_token_{id}|>"))
1067            .collect();
1068        let enc_multi = tokenizer.encode(&multi).unwrap();
1069        assert_eq!(
1070            enc_multi.token_ids().len(),
1071            10,
1072            "10 reserved token strings should produce exactly 10 tokens, got {}: {:?}",
1073            enc_multi.token_ids().len(),
1074            enc_multi.token_ids()
1075        );
1076        let expected_ids: Vec<u32> = (258..268).collect();
1077        assert_eq!(enc_multi.token_ids(), &expected_ids);
1078    }
1079
1080    /// Confirm that the old relative-offset naming would cause token inflation.
1081    /// Manually builds a tokenizer whose special-token map uses the WRONG names
1082    /// (relative offsets), then shows the same string encodes as many tokens.
1083    #[test]
1084    fn test_relative_offset_naming_causes_inflation() {
1085        let dir = tempfile::tempdir().unwrap();
1086        let file_path = create_byte_level_tiktoken_file(dir.path());
1087
1088        let _encoder = parse_tiktoken_file(&file_path).unwrap();
1089        let num_base_tokens = 256usize;
1090
1091        // Build special tokens with the OLD (buggy) relative-offset naming
1092        let mut bad_special_tokens: FxHashMap<String, u32> = FxHashMap::default();
1093        bad_special_tokens.insert("[BOS]".to_string(), 256);
1094        bad_special_tokens.insert("[EOS]".to_string(), 257);
1095        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
1096            let id = num_base_tokens as u32 + i;
1097            if id != 256 && id != 257 {
1098                // OLD naming: relative offset i, not absolute id
1099                bad_special_tokens.insert(format!("<|reserved_token_{i}|>"), id);
1100            }
1101        }
1102
1103        let bad_tokenizer =
1104            TikTokenTokenizer::from_file(&file_path, KIMI_PATTERN, bad_special_tokens).unwrap();
1105
1106        // With the wrong naming, <|reserved_token_258|> is NOT recognized as special.
1107        // It gets split into byte-level BPE tokens -> many more than 1.
1108        let input = "<|reserved_token_258|>";
1109        let enc = bad_tokenizer.encode(input).unwrap();
1110        assert!(
1111            enc.token_ids().len() > 1,
1112            "With buggy relative-offset naming, '{}' should NOT be recognized as a \
1113             single special token. Got {} token(s): {:?}",
1114            input,
1115            enc.token_ids().len(),
1116            enc.token_ids()
1117        );
1118
1119        // Show the inflation: 10 reserved tokens produce far more than 10 BPE tokens.
1120        let multi: String = (258u32..268)
1121            .map(|id| format!("<|reserved_token_{id}|>"))
1122            .collect();
1123        let enc_multi = bad_tokenizer.encode(&multi).unwrap();
1124        assert!(
1125            enc_multi.token_ids().len() > 10,
1126            "With buggy naming, 10 reserved token strings should inflate beyond 10 tokens. \
1127             Got {}",
1128            enc_multi.token_ids().len(),
1129        );
1130    }
1131}