Skip to main content

dynamo_tokenizers/
tiktoken.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::collections::HashSet;
5use std::path::Path;
6
7use base64::Engine as _;
8use rayon::prelude::*;
9use rustc_hash::FxHashMap;
10use tiktoken_rs::CoreBPE;
11
12use super::{
13    EncodeSegment, Encoding, Error, Result, TokenIdType,
14    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17/// Number of reserved special-token slots to generate when filling gaps in the vocabulary.
18/// Most tiktoken-based models reserve 256 IDs above the base vocabulary for special tokens.
19const DEFAULT_NUM_RESERVED_SPECIAL_TOKENS: u32 = 256;
20
21/// Kimi BPE pattern. `fastokens` takes its Kimi scanner fast path only for this exact string.
22const KIMI_PATTERN: &str = fastokens::tiktoken::KIMI_PATTERN;
23
24pub struct TikTokenTokenizer {
25    bpe: CoreBPE,
26    special_token_ids: HashSet<u32>,
27    special_tokens: Vec<String>,
28}
29
30pub(crate) fn sorted_special_token_strings(special_tokens: &FxHashMap<String, u32>) -> Vec<String> {
31    let mut strings: Vec<String> = special_tokens.keys().cloned().collect();
32    strings.sort();
33    strings.dedup();
34    strings
35}
36
37impl TikTokenTokenizer {
38    /// Create a TikTokenTokenizer from a tiktoken model file.
39    ///
40    /// # Arguments
41    /// * `path` - Path to the `.model` or `.tiktoken` file (base64 rank-per-line format)
42    /// * `pattern` - BPE regex pattern string
43    /// * `special_tokens` - Map of special token strings to their IDs
44    pub fn from_file(
45        path: &str,
46        pattern: &str,
47        special_tokens: FxHashMap<String, u32>,
48    ) -> Result<Self> {
49        let encoder = parse_tiktoken_file(path)?;
50        let special_token_ids: HashSet<u32> = special_tokens.values().copied().collect();
51        let special_token_strings = sorted_special_token_strings(&special_tokens);
52
53        let bpe = CoreBPE::new(encoder, special_tokens, pattern)
54            .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
55
56        Ok(Self {
57            bpe,
58            special_token_ids,
59            special_tokens: special_token_strings,
60        })
61    }
62
63    /// Create a TikTokenTokenizer from a tiktoken model file, auto-detecting
64    /// the BPE pattern from `config.json` and special tokens from `tokenizer_config.json`.
65    ///
66    /// The tiktoken file and config files must be in the same directory.
67    pub fn from_file_auto(path: &str) -> Result<Self> {
68        let file_path = Path::new(path);
69        let directory = file_path
70            .parent()
71            .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
72
73        let pattern = detect_bpe_pattern(directory)?;
74        let encoder = parse_tiktoken_file(path)?;
75        // Use max rank + 1 (not len) to avoid ID collisions with sparse/non-contiguous ranks
76        let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
77        let AddedTokens { tokens, skip_ids } = load_added_tokens(directory, num_base_tokens)?;
78        let special_token_strings = sorted_special_token_strings(&tokens);
79
80        let bpe = CoreBPE::new(encoder, tokens, pattern)
81            .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
82
83        Ok(Self {
84            bpe,
85            special_token_ids: skip_ids,
86            special_tokens: special_token_strings,
87        })
88    }
89
90    /// Atomic special-token strings registered with the underlying TikToken BPE.
91    /// Includes added tokens marked `"special": false`, unlike `special_token_ids`.
92    ///
93    /// [`CoreBPE::encode_with_special_tokens`] recognizes these strings outside ordinary
94    /// BPE. For non-overlapping token spellings, splitting immediately after a match
95    /// preserves tokenization.
96    pub fn special_tokens(&self) -> &[String] {
97        &self.special_tokens
98    }
99}
100
101impl Encoder for TikTokenTokenizer {
102    fn encode(&self, input: &str) -> Result<Encoding> {
103        let token_ids: Vec<u32> = self.bpe.encode_with_special_tokens(input);
104        Ok(Encoding::Sp(token_ids))
105    }
106
107    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
108        inputs.par_iter().map(|input| self.encode(input)).collect()
109    }
110
111    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
112        let mut token_ids = Vec::new();
113        for segment in segments {
114            for_each_reference_piece(segment.text, |piece| {
115                if segment.allow_special {
116                    token_ids.extend(self.bpe.encode_with_special_tokens(piece));
117                } else {
118                    token_ids.extend(self.bpe.encode_ordinary(piece));
119                }
120            });
121        }
122        Ok(Encoding::Sp(token_ids))
123    }
124}
125
126/// Longest text, in characters, the reference Kimi tokenizer passes to tiktoken at once.
127const MAX_ENCODE_CHARS: usize = 400_000;
128
129/// Longest run of whitespace or non-whitespace characters the reference Kimi tokenizer
130/// encodes in one piece (<https://github.com/openai/tiktoken/issues/195>).
131const MAX_RUN_CHARS: usize = 25_000;
132
133/// Python's `str.isspace`: Unicode `White_Space` plus the ASCII information separators
134/// U+001C..U+001F, which Rust's [`char::is_whitespace`] does not include.
135fn is_python_space(c: char) -> bool {
136    c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
137}
138
139/// Calls `f`, in order, on the pieces the reference Kimi tokenizer
140/// (`TikTokenTokenizer._encode_text_piece` in the checkpoint's `tokenization_kimi.py`)
141/// encodes separately: `text` cut into chunks of [`MAX_ENCODE_CHARS`] characters, each
142/// split before any whitespace or non-whitespace run would exceed [`MAX_RUN_CHARS`]
143/// characters. BPE does not merge across these cuts, so encoding the text whole yields
144/// different ids whenever a cut applies.
145fn for_each_reference_piece<'a>(text: &'a str, mut f: impl FnMut(&'a str)) {
146    // At most MAX_RUN_CHARS bytes means at most that many characters: no cut applies.
147    if text.len() <= MAX_RUN_CHARS {
148        f(text);
149        return;
150    }
151    let mut chunk_start = 0;
152    let mut chunk_chars = 0;
153    for (index, _) in text.char_indices() {
154        if chunk_chars == MAX_ENCODE_CHARS {
155            split_long_runs(&text[chunk_start..index], &mut f);
156            chunk_start = index;
157            chunk_chars = 0;
158        }
159        chunk_chars += 1;
160    }
161    split_long_runs(&text[chunk_start..], &mut f);
162}
163
164/// Calls `f` on `chunk` split before any whitespace or non-whitespace run would exceed
165/// [`MAX_RUN_CHARS`] characters.
166fn split_long_runs<'a>(chunk: &'a str, f: &mut impl FnMut(&'a str)) {
167    let mut run_is_space = chunk.chars().next().is_some_and(is_python_space);
168    let mut run_len = 0;
169    let mut piece_start = 0;
170    for (index, c) in chunk.char_indices() {
171        let is_space = is_python_space(c);
172        if is_space != run_is_space {
173            run_is_space = is_space;
174            run_len = 1;
175        } else {
176            run_len += 1;
177            if run_len > MAX_RUN_CHARS {
178                f(&chunk[piece_start..index]);
179                piece_start = index;
180                run_len = 1;
181            }
182        }
183    }
184    f(&chunk[piece_start..]);
185}
186
187impl Decoder for TikTokenTokenizer {
188    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
189        let ids: Vec<u32> = if skip_special_tokens {
190            token_ids
191                .iter()
192                .filter(|&&id| !self.special_token_ids.contains(&id))
193                .copied()
194                .collect()
195        } else {
196            token_ids.to_vec()
197        };
198
199        // Try strict UTF-8 first: valid bytes get `Complete` with zero extra allocation
200        // (takes ownership of the Vec). This correctly handles vocabulary tokens whose
201        // raw bytes are EF BF BD (legitimate U+FFFD) -- they are valid UTF-8 and must
202        // not be confused with incomplete multi-byte sequences.
203        //
204        // On failure, fall back to lossy conversion so partial multi-byte sequences
205        // become U+FFFD, then classify via the trailing-FFFD heuristic. This path is
206        // only hit during incremental detokenization of byte-fallback tokens.
207        let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
208        match String::from_utf8(bytes) {
209            Ok(text) => Ok(DecodeResult::Complete(text)),
210            Err(e) => {
211                let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
212                Ok(DecodeResult::from_decoded(text))
213            }
214        }
215    }
216}
217
218impl Tokenizer for TikTokenTokenizer {
219    fn validate_prefix_cache(&self) -> Result<()> {
220        Ok(())
221    }
222
223    // `tiktoken_rs::CoreBPE`'s `encoder` field is `pub(crate)`, so this
224    // crate has no way to enumerate, size, or look up ids in the vocab.
225    // `token_to_id` is left at the trait default (unsupported) for the
226    // same reason.
227    fn vocab_size(&self) -> Option<usize> {
228        None
229    }
230
231    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
232        let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
233        ids.sort_unstable();
234        Ok(ids)
235    }
236
237    // tiktoken applies no post-processor: `encode`'s `add_special_tokens`
238    // option (handled entirely by this crate, not tiktoken_rs) never adds
239    // tokens beyond what the caller passed in.
240    fn num_special_tokens_added(&self) -> Result<usize> {
241        Ok(0)
242    }
243}
244
245/// Parse a tiktoken model file (base64-encoded token + rank per line).
246pub(crate) fn parse_tiktoken_file(path: &str) -> Result<FxHashMap<Vec<u8>, u32>> {
247    let contents = std::fs::read_to_string(path)
248        .map_err(|err| Error::msg(format!("Failed to read tiktoken file '{path}': {err}")))?;
249
250    let engine = base64::engine::general_purpose::STANDARD;
251    let mut encoder = FxHashMap::default();
252
253    for line in contents.lines() {
254        let line = line.trim();
255        if line.is_empty() {
256            continue;
257        }
258        let mut parts = line.split_whitespace();
259        let token_b64 = parts
260            .next()
261            .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no token): {line}")))?;
262        let rank_str = parts
263            .next()
264            .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no rank): {line}")))?;
265
266        let token_bytes = engine
267            .decode(token_b64)
268            .map_err(|err| Error::msg(format!("Invalid base64 in tiktoken file: {err}")))?;
269        let rank: u32 = rank_str
270            .parse()
271            .map_err(|err| Error::msg(format!("Invalid rank in tiktoken file: {err}")))?;
272
273        encoder.insert(token_bytes, rank);
274    }
275
276    Ok(encoder)
277}
278
279/// Detect the BPE pattern for a model by reading `model_type` from `config.json`.
280pub(crate) fn detect_bpe_pattern(directory: &Path) -> Result<&'static str> {
281    let model_type: String = crate::file_json_field(&directory.join("config.json"), "model_type")
282        .map_err(|err| {
283        Error::msg(format!("Failed to read model_type from config.json: {err}"))
284    })?;
285
286    match model_type.as_str() {
287        // baseten-admin/Kimi-2.5-text-nvfp4-v3 model has model_type: "deepseek_v3" in its config.json
288        // because Kimi K2.5 is built on the DeepSeek V3 architecture.
289        // it still ships the Kimi tiktoken tokenizer file, so the KIMI_PATTERN BPE regex is the
290        // correct pattern to use.  No pure DeepSeek V3 model uses tiktoken.model files
291        // (they use tokenizer.json instead) so this match is safe.
292        "kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "kimi_linear" | "deepseek_v3" => {
293            Ok(KIMI_PATTERN)
294        }
295        _ => Err(Error::msg(format!(
296            "Unsupported tiktoken model_type '{model_type}'. \
297             Currently supported: kimi, kimi_k2, kimi_k25, kimi_k3, kimi_linear, deepseek_v3. \
298             To add a new model type, extend detect_bpe_pattern() in lib/tokenizers/src/tiktoken.rs \
299             with the appropriate BPE regex pattern. \
300             Alternatively, provide a tokenizer.json (HuggingFace format) instead."
301        ))),
302    }
303}
304
305/// Added tokens from `tokenizer_config.json`.
306pub(crate) struct AddedTokens {
307    /// Every added token; each encodes as one unit.
308    pub(crate) tokens: FxHashMap<String, u32>,
309    /// IDs `skip_special_tokens` removes: `"special": true` entries and reserved slots.
310    /// A missing `special` field counts as false, as in Hugging Face.
311    pub(crate) skip_ids: HashSet<u32>,
312}
313
314/// Load added tokens from `tokenizer_config.json` in the model directory.
315///
316/// Reads the `added_tokens_decoder` field which maps string token IDs to token definitions.
317/// Falls back to generating `<|reserved_token_{id}|>` names for unmapped IDs.
318pub(crate) fn load_added_tokens(directory: &Path, num_base_tokens: usize) -> Result<AddedTokens> {
319    let config_path = directory.join("tokenizer_config.json");
320    let mut tokens = FxHashMap::default();
321    let mut skip_ids = HashSet::new();
322
323    if !config_path.exists() {
324        // No tokenizer_config.json β€” generate default reserved tokens
325        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
326            let id = num_base_tokens as u32 + i;
327            tokens.insert(format!("<|reserved_token_{id}|>"), id);
328            skip_ids.insert(id);
329        }
330        return Ok(AddedTokens { tokens, skip_ids });
331    }
332
333    let contents = std::fs::read_to_string(&config_path)
334        .map_err(|err| Error::msg(format!("Failed to read tokenizer_config.json: {err}")))?;
335
336    let config: serde_json::Value = serde_json::from_str(&contents)
337        .map_err(|err| Error::msg(format!("Failed to parse tokenizer_config.json: {err}")))?;
338
339    if let Some(added_tokens) = config
340        .get("added_tokens_decoder")
341        .and_then(|v| v.as_object())
342    {
343        for (id_str, token_def) in added_tokens {
344            let id: u32 = id_str.parse().map_err(|err| {
345                Error::msg(format!(
346                    "Invalid token ID '{id_str}' in added_tokens_decoder: {err}"
347                ))
348            })?;
349
350            let content = token_def
351                .get("content")
352                .and_then(|v| v.as_str())
353                .unwrap_or_else(|| {
354                    // This shouldn't happen in well-formed configs, but handle gracefully
355                    tracing::warn!("Missing 'content' field for token ID {id}");
356                    ""
357                });
358
359            if !content.is_empty() {
360                tokens.insert(content.to_string(), id);
361                if token_def
362                    .get("special")
363                    .and_then(serde_json::Value::as_bool)
364                    == Some(true)
365                {
366                    skip_ids.insert(id);
367                }
368            }
369        }
370
371        // Fill in any gaps with reserved tokens for the expected range
372        let used_ids: HashSet<u32> = tokens.values().copied().collect();
373        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
374            let id = num_base_tokens as u32 + i;
375            if !used_ids.contains(&id) {
376                tokens.insert(format!("<|reserved_token_{id}|>"), id);
377                skip_ids.insert(id);
378            }
379        }
380    } else {
381        // No added_tokens_decoder β€” generate default reserved tokens
382        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
383            let id = num_base_tokens as u32 + i;
384            tokens.insert(format!("<|reserved_token_{id}|>"), id);
385            skip_ids.insert(id);
386        }
387    }
388
389    Ok(AddedTokens { tokens, skip_ids })
390}
391
392#[cfg(test)]
393mod tests {
394    use super::*;
395    use crate::DecodeStream;
396    use std::io::Write;
397    use std::sync::Arc;
398
399    fn create_test_tiktoken_file(dir: &Path) -> String {
400        let engine = base64::engine::general_purpose::STANDARD;
401        let mut content = String::new();
402
403        // Create some simple token entries: single bytes with sequential ranks
404        let tokens: Vec<(&[u8], u32)> = vec![
405            (b"h", 0),
406            (b"e", 1),
407            (b"l", 2),
408            (b"o", 3),
409            (b" ", 4),
410            (b"w", 5),
411            (b"r", 6),
412            (b"d", 7),
413            (b"he", 8),
414            (b"ll", 9),
415            (b"lo", 10),
416            (b"wo", 11),
417            (b"rl", 12),
418            (b"hel", 13),
419            (b"llo", 14),
420            (b"wor", 15),
421            (b"hell", 16),
422            (b"ello", 17),
423            (b"worl", 18),
424            (b"hello", 19),
425            (b"world", 20),
426        ];
427
428        for (token, rank) in tokens {
429            let encoded = engine.encode(token);
430            content.push_str(&format!("{encoded} {rank}\n"));
431        }
432
433        let file_path = dir.join("tiktoken.model");
434        let mut file = std::fs::File::create(&file_path).unwrap();
435        file.write_all(content.as_bytes()).unwrap();
436        file_path.to_str().unwrap().to_string()
437    }
438
439    fn create_test_config(dir: &Path, model_type: &str) {
440        let config = serde_json::json!({
441            "model_type": model_type,
442            "max_position_embeddings": 32768,
443            "eos_token_id": [21]
444        });
445        let file_path = dir.join("config.json");
446        std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
447    }
448
449    fn create_test_tokenizer_config(dir: &Path, num_base_tokens: usize) {
450        let mut added_tokens = serde_json::Map::new();
451        let bos_id = num_base_tokens;
452        let eos_id = num_base_tokens + 1;
453
454        added_tokens.insert(
455            bos_id.to_string(),
456            serde_json::json!({"content": "[BOS]", "special": true}),
457        );
458        added_tokens.insert(
459            eos_id.to_string(),
460            serde_json::json!({"content": "[EOS]", "special": true}),
461        );
462
463        let config = serde_json::json!({
464            "added_tokens_decoder": added_tokens
465        });
466
467        let file_path = dir.join("tokenizer_config.json");
468        std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
469    }
470
471    #[test]
472    fn test_parse_tiktoken_file() {
473        let dir = tempfile::tempdir().unwrap();
474        let file_path = create_test_tiktoken_file(dir.path());
475        let encoder = parse_tiktoken_file(&file_path).unwrap();
476        assert_eq!(encoder.len(), 21);
477        assert_eq!(encoder[b"hello".as_slice()], 19);
478        assert_eq!(encoder[b"world".as_slice()], 20);
479    }
480
481    #[test]
482    fn test_parse_tiktoken_file_missing() {
483        let result = parse_tiktoken_file("/nonexistent/path/tiktoken.model");
484        assert!(result.is_err());
485    }
486
487    #[test]
488    fn test_tiktoken_from_file() {
489        let dir = tempfile::tempdir().unwrap();
490        let file_path = create_test_tiktoken_file(dir.path());
491
492        let mut special_tokens = FxHashMap::default();
493        special_tokens.insert("[BOS]".to_string(), 21_u32);
494        special_tokens.insert("[EOS]".to_string(), 22_u32);
495
496        // Use a simple pattern for testing
497        let pattern = r"[\w]+|[^\w\s]+|\s+";
498
499        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
500
501        assert_eq!(
502            tokenizer.special_tokens(),
503            &["[BOS]".to_string(), "[EOS]".to_string()]
504        );
505
506        // Test encode
507        let encoding = tokenizer.encode("hello world").unwrap();
508        let ids = encoding.token_ids();
509        assert!(!ids.is_empty());
510
511        // Test decode roundtrip
512        let decoded: String = tokenizer.decode(ids, false).unwrap().into();
513        assert_eq!(decoded, "hello world");
514    }
515
516    #[test]
517    fn test_tiktoken_encoding_variant() {
518        let dir = tempfile::tempdir().unwrap();
519        let file_path = create_test_tiktoken_file(dir.path());
520
521        let special_tokens = FxHashMap::default();
522        let pattern = r"[\w]+|[^\w\s]+|\s+";
523
524        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
525        assert!(tokenizer.special_tokens().is_empty());
526        let encoding = tokenizer.encode("hello").unwrap();
527
528        // Verify it produces the Sp variant
529        match &encoding {
530            Encoding::Sp(_) => {}
531            other => panic!("Expected Encoding::Sp, got {:?}", other),
532        }
533    }
534
535    #[test]
536    fn test_tiktoken_skip_special_tokens() {
537        let dir = tempfile::tempdir().unwrap();
538        let file_path = create_test_tiktoken_file(dir.path());
539
540        let mut special_tokens = FxHashMap::default();
541        special_tokens.insert("[BOS]".to_string(), 21_u32);
542        special_tokens.insert("[EOS]".to_string(), 22_u32);
543
544        let pattern = r"[\w]+|[^\w\s]+|\s+";
545
546        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
547
548        // Encode hello and prepend/append special tokens
549        let encoding = tokenizer.encode("hello").unwrap();
550        let mut ids = vec![21u32]; // [BOS]
551        ids.extend(encoding.token_ids());
552        ids.push(22); // [EOS]
553
554        // Decode with skip_special_tokens=true should strip special tokens
555        let decoded_skip: String = tokenizer.decode(&ids, true).unwrap().into();
556        assert_eq!(decoded_skip, "hello");
557
558        // Decode with skip_special_tokens=false should include them
559        let decoded_all: String = tokenizer.decode(&ids, false).unwrap().into();
560        assert!(decoded_all.contains("hello"));
561    }
562
563    #[test]
564    fn test_special_token_ids_reports_configured_ids_sorted() {
565        let dir = tempfile::tempdir().unwrap();
566        let file_path = create_test_tiktoken_file(dir.path());
567
568        let mut special_tokens = FxHashMap::default();
569        special_tokens.insert("[EOS]".to_string(), 22_u32);
570        special_tokens.insert("[BOS]".to_string(), 21_u32);
571
572        let pattern = r"[\w]+|[^\w\s]+|\s+";
573        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
574
575        assert_eq!(tokenizer.special_token_ids().unwrap(), vec![21, 22]);
576    }
577
578    #[test]
579    fn test_vocab_size_and_token_to_id_are_unsupported() {
580        let dir = tempfile::tempdir().unwrap();
581        let file_path = create_test_tiktoken_file(dir.path());
582        let pattern = r"[\w]+|[^\w\s]+|\s+";
583        let tokenizer =
584            TikTokenTokenizer::from_file(&file_path, pattern, FxHashMap::default()).unwrap();
585
586        assert_eq!(tokenizer.vocab_size(), None);
587        assert!(tokenizer.token_to_id("hello").is_err());
588    }
589
590    #[test]
591    fn test_tiktoken_from_file_auto() {
592        let dir = tempfile::tempdir().unwrap();
593        let file_path = create_test_tiktoken_file(dir.path());
594
595        create_test_config(dir.path(), "kimi");
596        create_test_tokenizer_config(dir.path(), 21);
597
598        let mut expected_specials: Vec<String> = load_added_tokens(dir.path(), 21)
599            .unwrap()
600            .tokens
601            .into_keys()
602            .collect();
603        expected_specials.sort();
604        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
605        assert_eq!(tokenizer.special_tokens(), expected_specials);
606
607        // Basic encode/decode roundtrip
608        let encoding = tokenizer.encode("hello world").unwrap();
609        let ids = encoding.token_ids();
610        assert!(!ids.is_empty());
611
612        let decoded: String = tokenizer.decode(ids, false).unwrap().into();
613        assert_eq!(decoded, "hello world");
614    }
615
616    #[test]
617    fn test_detect_bpe_pattern_unknown() {
618        let dir = tempfile::tempdir().unwrap();
619        create_test_config(dir.path(), "unknown_model");
620        let result = detect_bpe_pattern(dir.path());
621        assert!(result.is_err());
622    }
623
624    #[test]
625    fn test_kimi_k3_uses_kimi_bpe_pattern() {
626        let dir = tempfile::tempdir().unwrap();
627        create_test_config(dir.path(), "kimi_k3");
628        assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
629    }
630
631    #[test]
632    fn test_detect_bpe_pattern_kimi_linear() {
633        let dir = tempfile::tempdir().unwrap();
634        create_test_config(dir.path(), "kimi_linear");
635        assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
636    }
637
638    #[test]
639    fn test_load_special_tokens_no_config() {
640        let dir = tempfile::tempdir().unwrap();
641        let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
642        assert_eq!(tokens.len(), 256);
643        assert_eq!(tokens["<|reserved_token_100|>"], 100);
644        assert_eq!(tokens["<|reserved_token_355|>"], 355);
645    }
646
647    #[test]
648    fn test_load_special_tokens_with_config() {
649        let dir = tempfile::tempdir().unwrap();
650        create_test_tokenizer_config(dir.path(), 100);
651        let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
652        assert_eq!(tokens["[BOS]"], 100);
653        assert_eq!(tokens["[EOS]"], 101);
654        // Should also have reserved tokens filling gaps
655        assert!(tokens.len() > 2);
656    }
657
658    /// Kimi-style `added_tokens_decoder`: a control token marked special, a format marker
659    /// marked `"special": false`, and one with no `special` field (Hugging Face default false).
660    fn create_tokenizer_with_non_special_added_tokens(dir: &Path) -> TikTokenTokenizer {
661        let file_path = create_test_tiktoken_file(dir);
662        create_test_config(dir, "kimi_k3");
663        let config = serde_json::json!({"added_tokens_decoder": {
664            "21": {"content": "[BOS]", "special": true},
665            "22": {"content": "<|open|>", "special": false},
666            "23": {"content": "<|sep|>"},
667        }});
668        std::fs::write(dir.join("tokenizer_config.json"), config.to_string()).unwrap();
669        TikTokenTokenizer::from_file_auto(&file_path).unwrap()
670    }
671
672    #[test]
673    fn test_skip_special_tokens_keeps_non_special_added_tokens() {
674        let dir = tempfile::tempdir().unwrap();
675        let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
676
677        let encoding = tokenizer.encode("<|open|>hello").unwrap();
678        assert_eq!(encoding.token_ids(), &[22, 19]);
679
680        let special_ids = tokenizer.special_token_ids().unwrap();
681        assert!(special_ids.contains(&21));
682        assert!(special_ids.contains(&24), "reserved placeholder");
683        assert!(!special_ids.contains(&22));
684        assert!(!special_ids.contains(&23));
685
686        let ids = [21, 22, 19, 23];
687        let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
688        assert_eq!(skipped, "<|open|>hello<|sep|>");
689        let kept: String = tokenizer.decode(&ids, false).unwrap().into();
690        assert_eq!(kept, "[BOS]<|open|>hello<|sep|>");
691    }
692
693    #[test]
694    fn test_decode_stream_emits_leading_non_special_added_token() {
695        let dir = tempfile::tempdir().unwrap();
696        let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
697        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
698
699        let mut stream = DecodeStream::new(tokenizer_arc, &[19], true);
700        assert_eq!(stream.step(22).unwrap().as_deref(), Some("<|open|>"));
701        assert_eq!(stream.step(19).unwrap().as_deref(), Some("hello"));
702        assert_eq!(stream.step(21).unwrap(), None);
703        assert_eq!(stream.step(23).unwrap().as_deref(), Some("<|sep|>"));
704    }
705
706    /// Helper: create a tiktoken file that includes raw byte tokens (byte fallback tokens).
707    fn create_test_tiktoken_file_with_byte_tokens(dir: &Path) -> String {
708        let engine = base64::engine::general_purpose::STANDARD;
709        let mut content = String::new();
710
711        let tokens: Vec<(&[u8], u32)> = vec![
712            (b"h", 0),
713            (b"e", 1),
714            (b"l", 2),
715            (b"o", 3),
716            (b" ", 4),
717            (b"hello", 5),
718        ];
719
720        for (token, rank) in &tokens {
721            let encoded = engine.encode(token);
722            content.push_str(&format!("{encoded} {rank}\n"));
723        }
724
725        // Byte-fallback tokens: individual bytes that form CJK character "δ½ " (U+4F60)
726        // UTF-8 encoding: 0xE4 0xBD 0xA0
727        let byte_tokens: Vec<(Vec<u8>, u32)> =
728            vec![(vec![0xE4], 100), (vec![0xBD], 101), (vec![0xA0], 102)];
729
730        for (token, rank) in &byte_tokens {
731            let encoded = engine.encode(token);
732            content.push_str(&format!("{encoded} {rank}\n"));
733        }
734
735        // Bytes for emoji "πŸ˜€" (U+1F600) β€” 4-byte UTF-8: 0xF0 0x9F 0x98 0x80
736        let emoji_tokens: Vec<(Vec<u8>, u32)> = vec![
737            (vec![0xF0], 200),
738            (vec![0x9F], 201),
739            (vec![0x98], 202),
740            (vec![0x80], 203),
741        ];
742
743        for (token, rank) in &emoji_tokens {
744            let encoded = engine.encode(token);
745            content.push_str(&format!("{encoded} {rank}\n"));
746        }
747
748        // Legitimate U+FFFD token: valid UTF-8 bytes EF BF BD (replacement character
749        // as an actual vocabulary entry, not an artifact of lossy conversion)
750        let fffd_token: Vec<(Vec<u8>, u32)> = vec![(vec![0xEF, 0xBF, 0xBD], 300)];
751
752        for (token, rank) in &fffd_token {
753            let encoded = engine.encode(token);
754            content.push_str(&format!("{encoded} {rank}\n"));
755        }
756
757        let file_path = dir.join("tiktoken.model");
758        let mut file = std::fs::File::create(&file_path).unwrap();
759        file.write_all(content.as_bytes()).unwrap();
760        file_path.to_str().unwrap().to_string()
761    }
762
763    fn create_byte_token_tokenizer(dir: &Path) -> TikTokenTokenizer {
764        let file_path = create_test_tiktoken_file_with_byte_tokens(dir);
765        let special_tokens = FxHashMap::default();
766        let pattern = r"[\w]+|[^\w\s]+|\s+";
767        TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap()
768    }
769
770    /// Reproduces the original panic: decoding a single byte-fallback token that is
771    /// part of a multi-byte UTF-8 character. Before the fix, CoreBPE::decode() would
772    /// call String::from_utf8() on [0xE4] and error with "incomplete utf-8 byte sequence".
773    #[test]
774    fn test_decode_single_incomplete_utf8_byte_does_not_error() {
775        let dir = tempfile::tempdir().unwrap();
776        let tokenizer = create_byte_token_tokenizer(dir.path());
777
778        let result = tokenizer.decode(&[100], false);
779        assert!(
780            result.is_ok(),
781            "decode() should not error on incomplete UTF-8 bytes"
782        );
783        let decode_result = result.unwrap();
784        assert!(
785            decode_result.is_partial(),
786            "incomplete UTF-8 byte should produce DecodeResult::Partial, got: {:?}",
787            decode_result
788        );
789    }
790
791    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
792    #[test]
793    fn test_decode_two_of_three_utf8_bytes_does_not_error() {
794        let dir = tempfile::tempdir().unwrap();
795        let tokenizer = create_byte_token_tokenizer(dir.path());
796
797        let result = tokenizer.decode(&[100, 101], false);
798        assert!(result.is_ok());
799        let decode_result = result.unwrap();
800        assert!(
801            decode_result.is_partial(),
802            "incomplete 2-of-3 UTF-8 bytes should produce DecodeResult::Partial, got: {:?}",
803            decode_result
804        );
805    }
806
807    /// When all bytes of a multi-byte character are present, the concatenated bytes form
808    /// valid UTF-8, so this test passes both before and after the fix. It serves as a
809    /// correctness check that the lossy conversion doesn't corrupt complete characters.
810    #[test]
811    fn test_decode_complete_multibyte_utf8_produces_correct_char() {
812        let dir = tempfile::tempdir().unwrap();
813        let tokenizer = create_byte_token_tokenizer(dir.path());
814
815        let result = tokenizer.decode(&[100, 101, 102], false);
816        assert!(result.is_ok());
817        assert_eq!(String::from(result.unwrap()), "δ½ ");
818    }
819
820    /// All 4 emoji bytes together form valid UTF-8, so this passes both before and after
821    /// the fix. Validates that lossy conversion doesn't alter complete multi-byte sequences.
822    #[test]
823    fn test_decode_complete_4byte_emoji_from_byte_tokens() {
824        let dir = tempfile::tempdir().unwrap();
825        let tokenizer = create_byte_token_tokenizer(dir.path());
826
827        let result = tokenizer.decode(&[200, 201, 202, 203], false);
828        assert!(result.is_ok());
829        assert_eq!(String::from(result.unwrap()), "πŸ˜€");
830    }
831
832    /// Regression test: a vocabulary token whose raw bytes are EF BF BD (the valid
833    /// UTF-8 encoding of U+FFFD) must decode as `Complete`, not `Partial`. Before the
834    /// from_utf8 fast-path fix, from_utf8_lossy + the trailing-FFFD heuristic would
835    /// misclassify this as Partial, causing the incremental decoder to suppress it.
836    #[test]
837    fn test_decode_legitimate_replacement_char_token_is_complete() {
838        let dir = tempfile::tempdir().unwrap();
839        let tokenizer = create_byte_token_tokenizer(dir.path());
840
841        let result = tokenizer.decode(&[300], false);
842        assert!(result.is_ok());
843        let decode_result = result.unwrap();
844        assert!(
845            decode_result.is_complete(),
846            "legitimate U+FFFD vocab token must be Complete, got: {:?}",
847            decode_result
848        );
849        assert_eq!(decode_result.as_str(), "\u{FFFD}");
850    }
851
852    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
853    #[test]
854    fn test_decode_partial_emoji_does_not_error() {
855        let dir = tempfile::tempdir().unwrap();
856        let tokenizer = create_byte_token_tokenizer(dir.path());
857
858        let result = tokenizer.decode(&[200], false);
859        assert!(result.is_ok());
860        assert!(result.unwrap().is_partial());
861    }
862
863    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
864    #[test]
865    fn test_decode_mixed_ascii_and_incomplete_bytes() {
866        let dir = tempfile::tempdir().unwrap();
867        let tokenizer = create_byte_token_tokenizer(dir.path());
868
869        let result = tokenizer.decode(&[5, 100], false);
870        assert!(result.is_ok());
871        let decode_result = result.unwrap();
872        assert!(
873            decode_result.is_partial(),
874            "trailing incomplete byte should produce DecodeResult::Partial"
875        );
876        let text: String = decode_result.into();
877        assert!(
878            text.starts_with("hello"),
879            "should start with 'hello', got: {:?}",
880            text
881        );
882    }
883
884    /// End-to-end incremental detokenization: DecodeStream buffers partial bytes,
885    /// emits the complete character once all bytes arrive.
886    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
887    #[test]
888    fn test_decode_stream_incremental_multibyte_reassembly() {
889        let dir = tempfile::tempdir().unwrap();
890        let tokenizer = create_byte_token_tokenizer(dir.path());
891        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
892
893        let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
894
895        let r1 = stream.step(100).unwrap();
896        assert_eq!(r1, None, "first byte of 3-byte char should be buffered");
897
898        let r2 = stream.step(101).unwrap();
899        assert_eq!(r2, None, "second byte of 3-byte char should be buffered");
900
901        let r3 = stream.step(102).unwrap();
902        assert!(r3.is_some(), "third byte should complete the character");
903        assert_eq!(r3.unwrap(), "δ½ ");
904    }
905
906    /// Without the fix, fails with "incomplete utf-8 byte sequence" from CoreBPE::decode().
907    #[test]
908    fn test_decode_stream_incremental_emoji_reassembly() {
909        let dir = tempfile::tempdir().unwrap();
910        let tokenizer = create_byte_token_tokenizer(dir.path());
911        let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
912
913        let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
914
915        let r1 = stream.step(200).unwrap();
916        assert_eq!(r1, None, "byte 1/4 of emoji should be buffered");
917
918        let r2 = stream.step(201).unwrap();
919        assert_eq!(r2, None, "byte 2/4 of emoji should be buffered");
920
921        let r3 = stream.step(202).unwrap();
922        assert_eq!(r3, None, "byte 3/4 of emoji should be buffered");
923
924        let r4 = stream.step(203).unwrap();
925        assert!(r4.is_some(), "byte 4/4 should complete the emoji");
926        assert_eq!(r4.unwrap(), "πŸ˜€");
927    }
928
929    #[test]
930    fn test_tiktoken_encode_batch() {
931        let dir = tempfile::tempdir().unwrap();
932        let file_path = create_test_tiktoken_file(dir.path());
933
934        let special_tokens = FxHashMap::default();
935        let pattern = r"[\w]+|[^\w\s]+|\s+";
936
937        let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
938
939        let inputs = &["hello", "world"];
940        let encodings = tokenizer.encode_batch(inputs).unwrap();
941        assert_eq!(encodings.len(), 2);
942
943        for (encoding, input) in encodings.iter().zip(inputs.iter()) {
944            let decoded: String = tokenizer
945                .decode(encoding.token_ids(), false)
946                .unwrap()
947                .into();
948            assert_eq!(decoded, *input);
949        }
950    }
951
952    /// Helper: create a tiktoken file containing all 256 single-byte tokens (ranks 0..255).
953    /// This gives a complete byte-level base vocabulary so any ASCII string can be encoded.
954    fn create_byte_level_tiktoken_file(dir: &Path) -> String {
955        let engine = base64::engine::general_purpose::STANDARD;
956        let mut content = String::new();
957        for byte_val in 0u16..256 {
958            let encoded = engine.encode([byte_val as u8]);
959            content.push_str(&format!("{encoded} {byte_val}\n"));
960        }
961        let file_path = dir.join("tiktoken.model");
962        std::fs::write(&file_path, &content).unwrap();
963        file_path.to_str().unwrap().to_string()
964    }
965
966    #[test]
967    fn test_segment_encoding_preserves_special_vs_ordinary_boundary() {
968        let dir = tempfile::tempdir().unwrap();
969        let file_path = create_byte_level_tiktoken_file(dir.path());
970        let marker = "<|open|>";
971        let marker_id = 256;
972        let mut special_tokens = FxHashMap::default();
973        special_tokens.insert(marker.to_string(), marker_id);
974        let tokenizer =
975            TikTokenTokenizer::from_file(&file_path, r"(?s).+", special_tokens).unwrap();
976
977        let structural = tokenizer
978            .encode_segments(&[EncodeSegment::control(marker)])
979            .unwrap();
980        let user_text = tokenizer
981            .encode_segments(&[EncodeSegment::ordinary(marker)])
982            .unwrap();
983
984        assert_eq!(structural.token_ids(), &[marker_id]);
985        assert_ne!(user_text.token_ids(), &[marker_id]);
986        let decoded: String = tokenizer
987            .decode(user_text.token_ids(), false)
988            .unwrap()
989            .into();
990        assert_eq!(decoded, marker);
991    }
992
993    /// Character lengths of [`for_each_reference_piece`]'s pieces, to compare with the
994    /// Python reference.
995    fn piece_chars(text: &str) -> Vec<usize> {
996        let mut lengths = Vec::new();
997        for_each_reference_piece(text, |piece| lengths.push(piece.chars().count()));
998        lengths
999    }
1000
1001    #[test]
1002    fn test_reference_pieces_match_kimi_reference_split() {
1003        // Expected lengths come from the checkpoint's `_encode_text_piece` split, run in
1004        // Python on the same strings.
1005        assert_eq!(piece_chars("hello world"), vec![11]);
1006        assert_eq!(
1007            piece_chars(&"a".repeat(60_000)),
1008            vec![25_000, 25_000, 10_000]
1009        );
1010        assert_eq!(
1011            piece_chars(&("a".repeat(30_000) + &" ".repeat(30_000))),
1012            vec![25_000, 30_000, 5_000]
1013        );
1014        assert_eq!(
1015            piece_chars(&("x".repeat(25_000) + " " + &"y".repeat(25_001))),
1016            vec![50_001, 1]
1017        );
1018        // Python's isspace counts U+001C..U+001F as whitespace, so the separators start a
1019        // new run; with `char::is_whitespace` this is one 30,000-character run, cut at
1020        // 25,000.
1021        assert_eq!(
1022            piece_chars(&("a".repeat(20_000) + &"\u{1c}".repeat(10_000))),
1023            vec![30_000]
1024        );
1025        // Lengths are in characters, not bytes.
1026        assert_eq!(piece_chars(&"Γ©".repeat(26_000)), vec![25_000, 1_000]);
1027        // Chunks of 400k characters, even without long runs.
1028        let chunked: String = "ab ".repeat(136_667).chars().take(410_000).collect();
1029        assert_eq!(piece_chars(&chunked), vec![400_000, 10_000]);
1030    }
1031
1032    #[test]
1033    fn test_segment_encoding_cuts_long_runs_like_the_reference() {
1034        // Byte vocabulary plus one merge ("aa"), so a cut inside an "aa" pair changes ids.
1035        let dir = tempfile::tempdir().unwrap();
1036        let engine = base64::engine::general_purpose::STANDARD;
1037        let mut content = String::new();
1038        for byte_val in 0u16..256 {
1039            let encoded = engine.encode([byte_val as u8]);
1040            content.push_str(&format!("{encoded} {byte_val}\n"));
1041        }
1042        content.push_str(&format!("{} 256\n", engine.encode(b"aa")));
1043        let file_path = dir.path().join("tiktoken.model");
1044        std::fs::write(&file_path, &content).unwrap();
1045        let tokenizer = TikTokenTokenizer::from_file(
1046            file_path.to_str().unwrap(),
1047            r"(?s).+",
1048            FxHashMap::default(),
1049        )
1050        .unwrap();
1051
1052        // One non-whitespace run of 25,001 characters: the reference encodes the first
1053        // 25,000 ("b" + 24,999 "a") and the last "a" separately.
1054        let text = "b".to_string() + &"a".repeat(25_000);
1055        let segmented = tokenizer
1056            .encode_segments(&[EncodeSegment::ordinary(&text)])
1057            .unwrap();
1058        let mut expected = tokenizer.bpe.encode_ordinary(&text[..25_000]);
1059        expected.extend(tokenizer.bpe.encode_ordinary(&text[25_000..]));
1060        assert_eq!(segmented.token_ids(), expected.as_slice());
1061        assert_ne!(
1062            segmented.token_ids(),
1063            tokenizer.bpe.encode_ordinary(&text).as_slice(),
1064            "the cut must change ids for this input, or the test proves nothing"
1065        );
1066    }
1067
1068    fn has_nontrivial_self_overlap(token: &str) -> bool {
1069        let bytes = token.as_bytes();
1070        (1..bytes.len()).any(|overlap| bytes[bytes.len() - overlap..] == bytes[..overlap])
1071    }
1072
1073    fn have_ambiguous_overlap(a: &str, b: &str) -> bool {
1074        let a = a.as_bytes();
1075        let b = b.as_bytes();
1076
1077        if a.windows(b.len()).any(|window| window == b)
1078            || b.windows(a.len()).any(|window| window == a)
1079        {
1080            return true;
1081        }
1082
1083        let max_overlap = a.len().min(b.len());
1084        (1..max_overlap).any(|overlap| {
1085            a[a.len() - overlap..] == b[..overlap] || b[b.len() - overlap..] == a[..overlap]
1086        })
1087    }
1088
1089    fn assert_special_split_invariant(
1090        tokenizer: &TikTokenTokenizer,
1091        left: &str,
1092        special: &str,
1093        right: &str,
1094    ) {
1095        let whole = tokenizer
1096            .encode(&format!("{left}{special}{right}"))
1097            .unwrap()
1098            .token_ids()
1099            .to_vec();
1100        let mut split = tokenizer
1101            .encode(&format!("{left}{special}"))
1102            .unwrap()
1103            .token_ids()
1104            .to_vec();
1105        split.extend_from_slice(tokenizer.encode(right).unwrap().token_ids());
1106        assert_eq!(
1107            whole, split,
1108            "splitting after registered special token {special:?} must preserve tokenization"
1109        );
1110    }
1111
1112    #[test]
1113    fn test_registered_special_tokens_are_cache_safe_boundaries() {
1114        let dir = tempfile::tempdir().unwrap();
1115        let file_path = create_byte_level_tiktoken_file(dir.path());
1116        create_test_config(dir.path(), "kimi");
1117        create_test_tokenizer_config(dir.path(), 256);
1118
1119        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1120        let specials = tokenizer.special_tokens();
1121        assert!(!specials.is_empty());
1122
1123        for (index, special) in specials.iter().enumerate() {
1124            assert!(
1125                !has_nontrivial_self_overlap(special),
1126                "registered special token has an ambiguous self-overlap: {special:?}"
1127            );
1128            for other in &specials[index + 1..] {
1129                assert!(
1130                    !have_ambiguous_overlap(special, other),
1131                    "registered special tokens overlap ambiguously: {special:?} and {other:?}"
1132                );
1133            }
1134
1135            for (left, right) in [
1136                ("ordinary prefix ", " ordinary suffix"),
1137                ("line before\n", "\tUnicode after: εŒ—δΊ¬ πŸ˜€"),
1138                ("", " trailing text"),
1139            ] {
1140                assert_special_split_invariant(&tokenizer, left, special, right);
1141            }
1142        }
1143
1144        let bos = specials
1145            .iter()
1146            .find(|token| token.as_str() == "[BOS]")
1147            .unwrap();
1148        let eos = specials
1149            .iter()
1150            .find(|token| token.as_str() == "[EOS]")
1151            .unwrap();
1152        assert_special_split_invariant(
1153            &tokenizer,
1154            "prefix ",
1155            bos,
1156            &format!("{eos}<|reserved_token_258|>Unicode 尾部"),
1157        );
1158    }
1159
1160    /// Regression test for Kimi K2.5 tokenizer inflation.
1161    ///
1162    /// Python's tokenization_kimi.py names unnamed reserved tokens by absolute ID:
1163    ///   `<|reserved_token_{absolute_id}|>`
1164    ///
1165    /// The Rust code previously used relative offsets (0..255) as the naming index,
1166    /// so when a prompt contained `<|reserved_token_258|>` the Rust tokenizer did NOT
1167    /// recognize it as a special token. Each occurrence was encoded as multiple BPE
1168    /// tokens instead of 1, inflating an 8192-token prompt to 9038 tokens and causing
1169    /// TRT-LLM to reject the request.
1170    #[test]
1171    fn test_reserved_token_absolute_id_naming_kimi_k25_regression() {
1172        let dir = tempfile::tempdir().unwrap();
1173        let file_path = create_byte_level_tiktoken_file(dir.path());
1174
1175        // config.json with kimi model type -> triggers KIMI_PATTERN
1176        create_test_config(dir.path(), "kimi");
1177
1178        // tokenizer_config.json: BOS at 256, EOS at 257. Base vocab is IDs 0..255.
1179        create_test_tokenizer_config(dir.path(), 256);
1180
1181        let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1182
1183        // ID 256 = [BOS], ID 257 = [EOS].
1184        // ID 258 = first UNNAMED reserved token.
1185        //   With fix:   named <|reserved_token_258|>  (absolute ID)
1186        //   Before fix: named <|reserved_token_2|>    (relative offset)
1187
1188        // Single unnamed reserved token should be recognized as 1 special token.
1189        let single = "<|reserved_token_258|>";
1190        let enc = tokenizer.encode(single).unwrap();
1191        assert_eq!(
1192            enc.token_ids().len(),
1193            1,
1194            "'{single}' should be 1 special token, got {} tokens: {:?}. \
1195             This means fallback naming still uses relative offsets instead of absolute IDs.",
1196            enc.token_ids().len(),
1197            enc.token_ids()
1198        );
1199        assert_eq!(enc.token_ids()[0], 258);
1200
1201        // Multiple unnamed reserved tokens in sequence (mini version of the benchmark).
1202        // IDs 258..268 are all unnamed; with the fix they're <|reserved_token_258|>..267.
1203        let multi: String = (258u32..268)
1204            .map(|id| format!("<|reserved_token_{id}|>"))
1205            .collect();
1206        let enc_multi = tokenizer.encode(&multi).unwrap();
1207        assert_eq!(
1208            enc_multi.token_ids().len(),
1209            10,
1210            "10 reserved token strings should produce exactly 10 tokens, got {}: {:?}",
1211            enc_multi.token_ids().len(),
1212            enc_multi.token_ids()
1213        );
1214        let expected_ids: Vec<u32> = (258..268).collect();
1215        assert_eq!(enc_multi.token_ids(), &expected_ids);
1216    }
1217
1218    /// Confirm that the old relative-offset naming would cause token inflation.
1219    /// Manually builds a tokenizer whose special-token map uses the WRONG names
1220    /// (relative offsets), then shows the same string encodes as many tokens.
1221    #[test]
1222    fn test_relative_offset_naming_causes_inflation() {
1223        let dir = tempfile::tempdir().unwrap();
1224        let file_path = create_byte_level_tiktoken_file(dir.path());
1225
1226        let _encoder = parse_tiktoken_file(&file_path).unwrap();
1227        let num_base_tokens = 256usize;
1228
1229        // Build special tokens with the OLD (buggy) relative-offset naming
1230        let mut bad_special_tokens: FxHashMap<String, u32> = FxHashMap::default();
1231        bad_special_tokens.insert("[BOS]".to_string(), 256);
1232        bad_special_tokens.insert("[EOS]".to_string(), 257);
1233        for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
1234            let id = num_base_tokens as u32 + i;
1235            if id != 256 && id != 257 {
1236                // OLD naming: relative offset i, not absolute id
1237                bad_special_tokens.insert(format!("<|reserved_token_{i}|>"), id);
1238            }
1239        }
1240
1241        let bad_tokenizer =
1242            TikTokenTokenizer::from_file(&file_path, KIMI_PATTERN, bad_special_tokens).unwrap();
1243
1244        // With the wrong naming, <|reserved_token_258|> is NOT recognized as special.
1245        // It gets split into byte-level BPE tokens -> many more than 1.
1246        let input = "<|reserved_token_258|>";
1247        let enc = bad_tokenizer.encode(input).unwrap();
1248        assert!(
1249            enc.token_ids().len() > 1,
1250            "With buggy relative-offset naming, '{}' should NOT be recognized as a \
1251             single special token. Got {} token(s): {:?}",
1252            input,
1253            enc.token_ids().len(),
1254            enc.token_ids()
1255        );
1256
1257        // Show the inflation: 10 reserved tokens produce far more than 10 BPE tokens.
1258        let multi: String = (258u32..268)
1259            .map(|id| format!("<|reserved_token_{id}|>"))
1260            .collect();
1261        let enc_multi = bad_tokenizer.encode(&multi).unwrap();
1262        assert!(
1263            enc_multi.token_ids().len() > 10,
1264            "With buggy naming, 10 reserved token strings should inflate beyond 10 tokens. \
1265             Got {}",
1266            enc_multi.token_ids().len(),
1267        );
1268    }
1269}