1use std::collections::HashSet;
5use std::path::Path;
6
7use base64::Engine as _;
8use rayon::prelude::*;
9use rustc_hash::FxHashMap;
10use tiktoken_rs::CoreBPE;
11
12use super::{
13 EncodeSegment, Encoding, Error, Result, TokenIdType,
14 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17const DEFAULT_NUM_RESERVED_SPECIAL_TOKENS: u32 = 256;
20
21const KIMI_PATTERN: &str = fastokens::tiktoken::KIMI_PATTERN;
23
24pub struct TikTokenTokenizer {
25 bpe: CoreBPE,
26 special_token_ids: HashSet<u32>,
27 special_tokens: Vec<String>,
28}
29
30pub(crate) fn sorted_special_token_strings(special_tokens: &FxHashMap<String, u32>) -> Vec<String> {
31 let mut strings: Vec<String> = special_tokens.keys().cloned().collect();
32 strings.sort();
33 strings.dedup();
34 strings
35}
36
37impl TikTokenTokenizer {
38 pub fn from_file(
45 path: &str,
46 pattern: &str,
47 special_tokens: FxHashMap<String, u32>,
48 ) -> Result<Self> {
49 let encoder = parse_tiktoken_file(path)?;
50 let special_token_ids: HashSet<u32> = special_tokens.values().copied().collect();
51 let special_token_strings = sorted_special_token_strings(&special_tokens);
52
53 let bpe = CoreBPE::new(encoder, special_tokens, pattern)
54 .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
55
56 Ok(Self {
57 bpe,
58 special_token_ids,
59 special_tokens: special_token_strings,
60 })
61 }
62
63 pub fn from_file_auto(path: &str) -> Result<Self> {
68 let file_path = Path::new(path);
69 let directory = file_path
70 .parent()
71 .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
72
73 let pattern = detect_bpe_pattern(directory)?;
74 let encoder = parse_tiktoken_file(path)?;
75 let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
77 let AddedTokens { tokens, skip_ids } = load_added_tokens(directory, num_base_tokens)?;
78 let special_token_strings = sorted_special_token_strings(&tokens);
79
80 let bpe = CoreBPE::new(encoder, tokens, pattern)
81 .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
82
83 Ok(Self {
84 bpe,
85 special_token_ids: skip_ids,
86 special_tokens: special_token_strings,
87 })
88 }
89
90 pub fn special_tokens(&self) -> &[String] {
97 &self.special_tokens
98 }
99}
100
101impl Encoder for TikTokenTokenizer {
102 fn encode(&self, input: &str) -> Result<Encoding> {
103 let token_ids: Vec<u32> = self.bpe.encode_with_special_tokens(input);
104 Ok(Encoding::Sp(token_ids))
105 }
106
107 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
108 inputs.par_iter().map(|input| self.encode(input)).collect()
109 }
110
111 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
112 let mut token_ids = Vec::new();
113 for segment in segments {
114 for_each_reference_piece(segment.text, |piece| {
115 if segment.allow_special {
116 token_ids.extend(self.bpe.encode_with_special_tokens(piece));
117 } else {
118 token_ids.extend(self.bpe.encode_ordinary(piece));
119 }
120 });
121 }
122 Ok(Encoding::Sp(token_ids))
123 }
124}
125
126const MAX_ENCODE_CHARS: usize = 400_000;
128
129const MAX_RUN_CHARS: usize = 25_000;
132
133fn is_python_space(c: char) -> bool {
136 c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
137}
138
139fn for_each_reference_piece<'a>(text: &'a str, mut f: impl FnMut(&'a str)) {
146 if text.len() <= MAX_RUN_CHARS {
148 f(text);
149 return;
150 }
151 let mut chunk_start = 0;
152 let mut chunk_chars = 0;
153 for (index, _) in text.char_indices() {
154 if chunk_chars == MAX_ENCODE_CHARS {
155 split_long_runs(&text[chunk_start..index], &mut f);
156 chunk_start = index;
157 chunk_chars = 0;
158 }
159 chunk_chars += 1;
160 }
161 split_long_runs(&text[chunk_start..], &mut f);
162}
163
164fn split_long_runs<'a>(chunk: &'a str, f: &mut impl FnMut(&'a str)) {
167 let mut run_is_space = chunk.chars().next().is_some_and(is_python_space);
168 let mut run_len = 0;
169 let mut piece_start = 0;
170 for (index, c) in chunk.char_indices() {
171 let is_space = is_python_space(c);
172 if is_space != run_is_space {
173 run_is_space = is_space;
174 run_len = 1;
175 } else {
176 run_len += 1;
177 if run_len > MAX_RUN_CHARS {
178 f(&chunk[piece_start..index]);
179 piece_start = index;
180 run_len = 1;
181 }
182 }
183 }
184 f(&chunk[piece_start..]);
185}
186
187impl Decoder for TikTokenTokenizer {
188 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
189 let ids: Vec<u32> = if skip_special_tokens {
190 token_ids
191 .iter()
192 .filter(|&&id| !self.special_token_ids.contains(&id))
193 .copied()
194 .collect()
195 } else {
196 token_ids.to_vec()
197 };
198
199 let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
208 match String::from_utf8(bytes) {
209 Ok(text) => Ok(DecodeResult::Complete(text)),
210 Err(e) => {
211 let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
212 Ok(DecodeResult::from_decoded(text))
213 }
214 }
215 }
216}
217
218impl Tokenizer for TikTokenTokenizer {
219 fn validate_prefix_cache(&self) -> Result<()> {
220 Ok(())
221 }
222
223 fn vocab_size(&self) -> Option<usize> {
228 None
229 }
230
231 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
232 let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
233 ids.sort_unstable();
234 Ok(ids)
235 }
236
237 fn num_special_tokens_added(&self) -> Result<usize> {
241 Ok(0)
242 }
243}
244
245pub(crate) fn parse_tiktoken_file(path: &str) -> Result<FxHashMap<Vec<u8>, u32>> {
247 let contents = std::fs::read_to_string(path)
248 .map_err(|err| Error::msg(format!("Failed to read tiktoken file '{path}': {err}")))?;
249
250 let engine = base64::engine::general_purpose::STANDARD;
251 let mut encoder = FxHashMap::default();
252
253 for line in contents.lines() {
254 let line = line.trim();
255 if line.is_empty() {
256 continue;
257 }
258 let mut parts = line.split_whitespace();
259 let token_b64 = parts
260 .next()
261 .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no token): {line}")))?;
262 let rank_str = parts
263 .next()
264 .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no rank): {line}")))?;
265
266 let token_bytes = engine
267 .decode(token_b64)
268 .map_err(|err| Error::msg(format!("Invalid base64 in tiktoken file: {err}")))?;
269 let rank: u32 = rank_str
270 .parse()
271 .map_err(|err| Error::msg(format!("Invalid rank in tiktoken file: {err}")))?;
272
273 encoder.insert(token_bytes, rank);
274 }
275
276 Ok(encoder)
277}
278
279pub(crate) fn detect_bpe_pattern(directory: &Path) -> Result<&'static str> {
281 let model_type: String = crate::file_json_field(&directory.join("config.json"), "model_type")
282 .map_err(|err| {
283 Error::msg(format!("Failed to read model_type from config.json: {err}"))
284 })?;
285
286 match model_type.as_str() {
287 "kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "kimi_linear" | "deepseek_v3" => {
293 Ok(KIMI_PATTERN)
294 }
295 _ => Err(Error::msg(format!(
296 "Unsupported tiktoken model_type '{model_type}'. \
297 Currently supported: kimi, kimi_k2, kimi_k25, kimi_k3, kimi_linear, deepseek_v3. \
298 To add a new model type, extend detect_bpe_pattern() in lib/tokenizers/src/tiktoken.rs \
299 with the appropriate BPE regex pattern. \
300 Alternatively, provide a tokenizer.json (HuggingFace format) instead."
301 ))),
302 }
303}
304
305pub(crate) struct AddedTokens {
307 pub(crate) tokens: FxHashMap<String, u32>,
309 pub(crate) skip_ids: HashSet<u32>,
312}
313
314pub(crate) fn load_added_tokens(directory: &Path, num_base_tokens: usize) -> Result<AddedTokens> {
319 let config_path = directory.join("tokenizer_config.json");
320 let mut tokens = FxHashMap::default();
321 let mut skip_ids = HashSet::new();
322
323 if !config_path.exists() {
324 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
326 let id = num_base_tokens as u32 + i;
327 tokens.insert(format!("<|reserved_token_{id}|>"), id);
328 skip_ids.insert(id);
329 }
330 return Ok(AddedTokens { tokens, skip_ids });
331 }
332
333 let contents = std::fs::read_to_string(&config_path)
334 .map_err(|err| Error::msg(format!("Failed to read tokenizer_config.json: {err}")))?;
335
336 let config: serde_json::Value = serde_json::from_str(&contents)
337 .map_err(|err| Error::msg(format!("Failed to parse tokenizer_config.json: {err}")))?;
338
339 if let Some(added_tokens) = config
340 .get("added_tokens_decoder")
341 .and_then(|v| v.as_object())
342 {
343 for (id_str, token_def) in added_tokens {
344 let id: u32 = id_str.parse().map_err(|err| {
345 Error::msg(format!(
346 "Invalid token ID '{id_str}' in added_tokens_decoder: {err}"
347 ))
348 })?;
349
350 let content = token_def
351 .get("content")
352 .and_then(|v| v.as_str())
353 .unwrap_or_else(|| {
354 tracing::warn!("Missing 'content' field for token ID {id}");
356 ""
357 });
358
359 if !content.is_empty() {
360 tokens.insert(content.to_string(), id);
361 if token_def
362 .get("special")
363 .and_then(serde_json::Value::as_bool)
364 == Some(true)
365 {
366 skip_ids.insert(id);
367 }
368 }
369 }
370
371 let used_ids: HashSet<u32> = tokens.values().copied().collect();
373 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
374 let id = num_base_tokens as u32 + i;
375 if !used_ids.contains(&id) {
376 tokens.insert(format!("<|reserved_token_{id}|>"), id);
377 skip_ids.insert(id);
378 }
379 }
380 } else {
381 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
383 let id = num_base_tokens as u32 + i;
384 tokens.insert(format!("<|reserved_token_{id}|>"), id);
385 skip_ids.insert(id);
386 }
387 }
388
389 Ok(AddedTokens { tokens, skip_ids })
390}
391
392#[cfg(test)]
393mod tests {
394 use super::*;
395 use crate::DecodeStream;
396 use std::io::Write;
397 use std::sync::Arc;
398
399 fn create_test_tiktoken_file(dir: &Path) -> String {
400 let engine = base64::engine::general_purpose::STANDARD;
401 let mut content = String::new();
402
403 let tokens: Vec<(&[u8], u32)> = vec![
405 (b"h", 0),
406 (b"e", 1),
407 (b"l", 2),
408 (b"o", 3),
409 (b" ", 4),
410 (b"w", 5),
411 (b"r", 6),
412 (b"d", 7),
413 (b"he", 8),
414 (b"ll", 9),
415 (b"lo", 10),
416 (b"wo", 11),
417 (b"rl", 12),
418 (b"hel", 13),
419 (b"llo", 14),
420 (b"wor", 15),
421 (b"hell", 16),
422 (b"ello", 17),
423 (b"worl", 18),
424 (b"hello", 19),
425 (b"world", 20),
426 ];
427
428 for (token, rank) in tokens {
429 let encoded = engine.encode(token);
430 content.push_str(&format!("{encoded} {rank}\n"));
431 }
432
433 let file_path = dir.join("tiktoken.model");
434 let mut file = std::fs::File::create(&file_path).unwrap();
435 file.write_all(content.as_bytes()).unwrap();
436 file_path.to_str().unwrap().to_string()
437 }
438
439 fn create_test_config(dir: &Path, model_type: &str) {
440 let config = serde_json::json!({
441 "model_type": model_type,
442 "max_position_embeddings": 32768,
443 "eos_token_id": [21]
444 });
445 let file_path = dir.join("config.json");
446 std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
447 }
448
449 fn create_test_tokenizer_config(dir: &Path, num_base_tokens: usize) {
450 let mut added_tokens = serde_json::Map::new();
451 let bos_id = num_base_tokens;
452 let eos_id = num_base_tokens + 1;
453
454 added_tokens.insert(
455 bos_id.to_string(),
456 serde_json::json!({"content": "[BOS]", "special": true}),
457 );
458 added_tokens.insert(
459 eos_id.to_string(),
460 serde_json::json!({"content": "[EOS]", "special": true}),
461 );
462
463 let config = serde_json::json!({
464 "added_tokens_decoder": added_tokens
465 });
466
467 let file_path = dir.join("tokenizer_config.json");
468 std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
469 }
470
471 #[test]
472 fn test_parse_tiktoken_file() {
473 let dir = tempfile::tempdir().unwrap();
474 let file_path = create_test_tiktoken_file(dir.path());
475 let encoder = parse_tiktoken_file(&file_path).unwrap();
476 assert_eq!(encoder.len(), 21);
477 assert_eq!(encoder[b"hello".as_slice()], 19);
478 assert_eq!(encoder[b"world".as_slice()], 20);
479 }
480
481 #[test]
482 fn test_parse_tiktoken_file_missing() {
483 let result = parse_tiktoken_file("/nonexistent/path/tiktoken.model");
484 assert!(result.is_err());
485 }
486
487 #[test]
488 fn test_tiktoken_from_file() {
489 let dir = tempfile::tempdir().unwrap();
490 let file_path = create_test_tiktoken_file(dir.path());
491
492 let mut special_tokens = FxHashMap::default();
493 special_tokens.insert("[BOS]".to_string(), 21_u32);
494 special_tokens.insert("[EOS]".to_string(), 22_u32);
495
496 let pattern = r"[\w]+|[^\w\s]+|\s+";
498
499 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
500
501 assert_eq!(
502 tokenizer.special_tokens(),
503 &["[BOS]".to_string(), "[EOS]".to_string()]
504 );
505
506 let encoding = tokenizer.encode("hello world").unwrap();
508 let ids = encoding.token_ids();
509 assert!(!ids.is_empty());
510
511 let decoded: String = tokenizer.decode(ids, false).unwrap().into();
513 assert_eq!(decoded, "hello world");
514 }
515
516 #[test]
517 fn test_tiktoken_encoding_variant() {
518 let dir = tempfile::tempdir().unwrap();
519 let file_path = create_test_tiktoken_file(dir.path());
520
521 let special_tokens = FxHashMap::default();
522 let pattern = r"[\w]+|[^\w\s]+|\s+";
523
524 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
525 assert!(tokenizer.special_tokens().is_empty());
526 let encoding = tokenizer.encode("hello").unwrap();
527
528 match &encoding {
530 Encoding::Sp(_) => {}
531 other => panic!("Expected Encoding::Sp, got {:?}", other),
532 }
533 }
534
535 #[test]
536 fn test_tiktoken_skip_special_tokens() {
537 let dir = tempfile::tempdir().unwrap();
538 let file_path = create_test_tiktoken_file(dir.path());
539
540 let mut special_tokens = FxHashMap::default();
541 special_tokens.insert("[BOS]".to_string(), 21_u32);
542 special_tokens.insert("[EOS]".to_string(), 22_u32);
543
544 let pattern = r"[\w]+|[^\w\s]+|\s+";
545
546 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
547
548 let encoding = tokenizer.encode("hello").unwrap();
550 let mut ids = vec![21u32]; ids.extend(encoding.token_ids());
552 ids.push(22); let decoded_skip: String = tokenizer.decode(&ids, true).unwrap().into();
556 assert_eq!(decoded_skip, "hello");
557
558 let decoded_all: String = tokenizer.decode(&ids, false).unwrap().into();
560 assert!(decoded_all.contains("hello"));
561 }
562
563 #[test]
564 fn test_special_token_ids_reports_configured_ids_sorted() {
565 let dir = tempfile::tempdir().unwrap();
566 let file_path = create_test_tiktoken_file(dir.path());
567
568 let mut special_tokens = FxHashMap::default();
569 special_tokens.insert("[EOS]".to_string(), 22_u32);
570 special_tokens.insert("[BOS]".to_string(), 21_u32);
571
572 let pattern = r"[\w]+|[^\w\s]+|\s+";
573 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
574
575 assert_eq!(tokenizer.special_token_ids().unwrap(), vec![21, 22]);
576 }
577
578 #[test]
579 fn test_vocab_size_and_token_to_id_are_unsupported() {
580 let dir = tempfile::tempdir().unwrap();
581 let file_path = create_test_tiktoken_file(dir.path());
582 let pattern = r"[\w]+|[^\w\s]+|\s+";
583 let tokenizer =
584 TikTokenTokenizer::from_file(&file_path, pattern, FxHashMap::default()).unwrap();
585
586 assert_eq!(tokenizer.vocab_size(), None);
587 assert!(tokenizer.token_to_id("hello").is_err());
588 }
589
590 #[test]
591 fn test_tiktoken_from_file_auto() {
592 let dir = tempfile::tempdir().unwrap();
593 let file_path = create_test_tiktoken_file(dir.path());
594
595 create_test_config(dir.path(), "kimi");
596 create_test_tokenizer_config(dir.path(), 21);
597
598 let mut expected_specials: Vec<String> = load_added_tokens(dir.path(), 21)
599 .unwrap()
600 .tokens
601 .into_keys()
602 .collect();
603 expected_specials.sort();
604 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
605 assert_eq!(tokenizer.special_tokens(), expected_specials);
606
607 let encoding = tokenizer.encode("hello world").unwrap();
609 let ids = encoding.token_ids();
610 assert!(!ids.is_empty());
611
612 let decoded: String = tokenizer.decode(ids, false).unwrap().into();
613 assert_eq!(decoded, "hello world");
614 }
615
616 #[test]
617 fn test_detect_bpe_pattern_unknown() {
618 let dir = tempfile::tempdir().unwrap();
619 create_test_config(dir.path(), "unknown_model");
620 let result = detect_bpe_pattern(dir.path());
621 assert!(result.is_err());
622 }
623
624 #[test]
625 fn test_kimi_k3_uses_kimi_bpe_pattern() {
626 let dir = tempfile::tempdir().unwrap();
627 create_test_config(dir.path(), "kimi_k3");
628 assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
629 }
630
631 #[test]
632 fn test_detect_bpe_pattern_kimi_linear() {
633 let dir = tempfile::tempdir().unwrap();
634 create_test_config(dir.path(), "kimi_linear");
635 assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
636 }
637
638 #[test]
639 fn test_load_special_tokens_no_config() {
640 let dir = tempfile::tempdir().unwrap();
641 let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
642 assert_eq!(tokens.len(), 256);
643 assert_eq!(tokens["<|reserved_token_100|>"], 100);
644 assert_eq!(tokens["<|reserved_token_355|>"], 355);
645 }
646
647 #[test]
648 fn test_load_special_tokens_with_config() {
649 let dir = tempfile::tempdir().unwrap();
650 create_test_tokenizer_config(dir.path(), 100);
651 let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
652 assert_eq!(tokens["[BOS]"], 100);
653 assert_eq!(tokens["[EOS]"], 101);
654 assert!(tokens.len() > 2);
656 }
657
658 fn create_tokenizer_with_non_special_added_tokens(dir: &Path) -> TikTokenTokenizer {
661 let file_path = create_test_tiktoken_file(dir);
662 create_test_config(dir, "kimi_k3");
663 let config = serde_json::json!({"added_tokens_decoder": {
664 "21": {"content": "[BOS]", "special": true},
665 "22": {"content": "<|open|>", "special": false},
666 "23": {"content": "<|sep|>"},
667 }});
668 std::fs::write(dir.join("tokenizer_config.json"), config.to_string()).unwrap();
669 TikTokenTokenizer::from_file_auto(&file_path).unwrap()
670 }
671
672 #[test]
673 fn test_skip_special_tokens_keeps_non_special_added_tokens() {
674 let dir = tempfile::tempdir().unwrap();
675 let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
676
677 let encoding = tokenizer.encode("<|open|>hello").unwrap();
678 assert_eq!(encoding.token_ids(), &[22, 19]);
679
680 let special_ids = tokenizer.special_token_ids().unwrap();
681 assert!(special_ids.contains(&21));
682 assert!(special_ids.contains(&24), "reserved placeholder");
683 assert!(!special_ids.contains(&22));
684 assert!(!special_ids.contains(&23));
685
686 let ids = [21, 22, 19, 23];
687 let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
688 assert_eq!(skipped, "<|open|>hello<|sep|>");
689 let kept: String = tokenizer.decode(&ids, false).unwrap().into();
690 assert_eq!(kept, "[BOS]<|open|>hello<|sep|>");
691 }
692
693 #[test]
694 fn test_decode_stream_emits_leading_non_special_added_token() {
695 let dir = tempfile::tempdir().unwrap();
696 let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
697 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
698
699 let mut stream = DecodeStream::new(tokenizer_arc, &[19], true);
700 assert_eq!(stream.step(22).unwrap().as_deref(), Some("<|open|>"));
701 assert_eq!(stream.step(19).unwrap().as_deref(), Some("hello"));
702 assert_eq!(stream.step(21).unwrap(), None);
703 assert_eq!(stream.step(23).unwrap().as_deref(), Some("<|sep|>"));
704 }
705
706 fn create_test_tiktoken_file_with_byte_tokens(dir: &Path) -> String {
708 let engine = base64::engine::general_purpose::STANDARD;
709 let mut content = String::new();
710
711 let tokens: Vec<(&[u8], u32)> = vec![
712 (b"h", 0),
713 (b"e", 1),
714 (b"l", 2),
715 (b"o", 3),
716 (b" ", 4),
717 (b"hello", 5),
718 ];
719
720 for (token, rank) in &tokens {
721 let encoded = engine.encode(token);
722 content.push_str(&format!("{encoded} {rank}\n"));
723 }
724
725 let byte_tokens: Vec<(Vec<u8>, u32)> =
728 vec![(vec![0xE4], 100), (vec![0xBD], 101), (vec![0xA0], 102)];
729
730 for (token, rank) in &byte_tokens {
731 let encoded = engine.encode(token);
732 content.push_str(&format!("{encoded} {rank}\n"));
733 }
734
735 let emoji_tokens: Vec<(Vec<u8>, u32)> = vec![
737 (vec![0xF0], 200),
738 (vec![0x9F], 201),
739 (vec![0x98], 202),
740 (vec![0x80], 203),
741 ];
742
743 for (token, rank) in &emoji_tokens {
744 let encoded = engine.encode(token);
745 content.push_str(&format!("{encoded} {rank}\n"));
746 }
747
748 let fffd_token: Vec<(Vec<u8>, u32)> = vec![(vec![0xEF, 0xBF, 0xBD], 300)];
751
752 for (token, rank) in &fffd_token {
753 let encoded = engine.encode(token);
754 content.push_str(&format!("{encoded} {rank}\n"));
755 }
756
757 let file_path = dir.join("tiktoken.model");
758 let mut file = std::fs::File::create(&file_path).unwrap();
759 file.write_all(content.as_bytes()).unwrap();
760 file_path.to_str().unwrap().to_string()
761 }
762
763 fn create_byte_token_tokenizer(dir: &Path) -> TikTokenTokenizer {
764 let file_path = create_test_tiktoken_file_with_byte_tokens(dir);
765 let special_tokens = FxHashMap::default();
766 let pattern = r"[\w]+|[^\w\s]+|\s+";
767 TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap()
768 }
769
770 #[test]
774 fn test_decode_single_incomplete_utf8_byte_does_not_error() {
775 let dir = tempfile::tempdir().unwrap();
776 let tokenizer = create_byte_token_tokenizer(dir.path());
777
778 let result = tokenizer.decode(&[100], false);
779 assert!(
780 result.is_ok(),
781 "decode() should not error on incomplete UTF-8 bytes"
782 );
783 let decode_result = result.unwrap();
784 assert!(
785 decode_result.is_partial(),
786 "incomplete UTF-8 byte should produce DecodeResult::Partial, got: {:?}",
787 decode_result
788 );
789 }
790
791 #[test]
793 fn test_decode_two_of_three_utf8_bytes_does_not_error() {
794 let dir = tempfile::tempdir().unwrap();
795 let tokenizer = create_byte_token_tokenizer(dir.path());
796
797 let result = tokenizer.decode(&[100, 101], false);
798 assert!(result.is_ok());
799 let decode_result = result.unwrap();
800 assert!(
801 decode_result.is_partial(),
802 "incomplete 2-of-3 UTF-8 bytes should produce DecodeResult::Partial, got: {:?}",
803 decode_result
804 );
805 }
806
807 #[test]
811 fn test_decode_complete_multibyte_utf8_produces_correct_char() {
812 let dir = tempfile::tempdir().unwrap();
813 let tokenizer = create_byte_token_tokenizer(dir.path());
814
815 let result = tokenizer.decode(&[100, 101, 102], false);
816 assert!(result.is_ok());
817 assert_eq!(String::from(result.unwrap()), "δ½ ");
818 }
819
820 #[test]
823 fn test_decode_complete_4byte_emoji_from_byte_tokens() {
824 let dir = tempfile::tempdir().unwrap();
825 let tokenizer = create_byte_token_tokenizer(dir.path());
826
827 let result = tokenizer.decode(&[200, 201, 202, 203], false);
828 assert!(result.is_ok());
829 assert_eq!(String::from(result.unwrap()), "π");
830 }
831
832 #[test]
837 fn test_decode_legitimate_replacement_char_token_is_complete() {
838 let dir = tempfile::tempdir().unwrap();
839 let tokenizer = create_byte_token_tokenizer(dir.path());
840
841 let result = tokenizer.decode(&[300], false);
842 assert!(result.is_ok());
843 let decode_result = result.unwrap();
844 assert!(
845 decode_result.is_complete(),
846 "legitimate U+FFFD vocab token must be Complete, got: {:?}",
847 decode_result
848 );
849 assert_eq!(decode_result.as_str(), "\u{FFFD}");
850 }
851
852 #[test]
854 fn test_decode_partial_emoji_does_not_error() {
855 let dir = tempfile::tempdir().unwrap();
856 let tokenizer = create_byte_token_tokenizer(dir.path());
857
858 let result = tokenizer.decode(&[200], false);
859 assert!(result.is_ok());
860 assert!(result.unwrap().is_partial());
861 }
862
863 #[test]
865 fn test_decode_mixed_ascii_and_incomplete_bytes() {
866 let dir = tempfile::tempdir().unwrap();
867 let tokenizer = create_byte_token_tokenizer(dir.path());
868
869 let result = tokenizer.decode(&[5, 100], false);
870 assert!(result.is_ok());
871 let decode_result = result.unwrap();
872 assert!(
873 decode_result.is_partial(),
874 "trailing incomplete byte should produce DecodeResult::Partial"
875 );
876 let text: String = decode_result.into();
877 assert!(
878 text.starts_with("hello"),
879 "should start with 'hello', got: {:?}",
880 text
881 );
882 }
883
884 #[test]
888 fn test_decode_stream_incremental_multibyte_reassembly() {
889 let dir = tempfile::tempdir().unwrap();
890 let tokenizer = create_byte_token_tokenizer(dir.path());
891 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
892
893 let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
894
895 let r1 = stream.step(100).unwrap();
896 assert_eq!(r1, None, "first byte of 3-byte char should be buffered");
897
898 let r2 = stream.step(101).unwrap();
899 assert_eq!(r2, None, "second byte of 3-byte char should be buffered");
900
901 let r3 = stream.step(102).unwrap();
902 assert!(r3.is_some(), "third byte should complete the character");
903 assert_eq!(r3.unwrap(), "δ½ ");
904 }
905
906 #[test]
908 fn test_decode_stream_incremental_emoji_reassembly() {
909 let dir = tempfile::tempdir().unwrap();
910 let tokenizer = create_byte_token_tokenizer(dir.path());
911 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
912
913 let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
914
915 let r1 = stream.step(200).unwrap();
916 assert_eq!(r1, None, "byte 1/4 of emoji should be buffered");
917
918 let r2 = stream.step(201).unwrap();
919 assert_eq!(r2, None, "byte 2/4 of emoji should be buffered");
920
921 let r3 = stream.step(202).unwrap();
922 assert_eq!(r3, None, "byte 3/4 of emoji should be buffered");
923
924 let r4 = stream.step(203).unwrap();
925 assert!(r4.is_some(), "byte 4/4 should complete the emoji");
926 assert_eq!(r4.unwrap(), "π");
927 }
928
929 #[test]
930 fn test_tiktoken_encode_batch() {
931 let dir = tempfile::tempdir().unwrap();
932 let file_path = create_test_tiktoken_file(dir.path());
933
934 let special_tokens = FxHashMap::default();
935 let pattern = r"[\w]+|[^\w\s]+|\s+";
936
937 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
938
939 let inputs = &["hello", "world"];
940 let encodings = tokenizer.encode_batch(inputs).unwrap();
941 assert_eq!(encodings.len(), 2);
942
943 for (encoding, input) in encodings.iter().zip(inputs.iter()) {
944 let decoded: String = tokenizer
945 .decode(encoding.token_ids(), false)
946 .unwrap()
947 .into();
948 assert_eq!(decoded, *input);
949 }
950 }
951
952 fn create_byte_level_tiktoken_file(dir: &Path) -> String {
955 let engine = base64::engine::general_purpose::STANDARD;
956 let mut content = String::new();
957 for byte_val in 0u16..256 {
958 let encoded = engine.encode([byte_val as u8]);
959 content.push_str(&format!("{encoded} {byte_val}\n"));
960 }
961 let file_path = dir.join("tiktoken.model");
962 std::fs::write(&file_path, &content).unwrap();
963 file_path.to_str().unwrap().to_string()
964 }
965
966 #[test]
967 fn test_segment_encoding_preserves_special_vs_ordinary_boundary() {
968 let dir = tempfile::tempdir().unwrap();
969 let file_path = create_byte_level_tiktoken_file(dir.path());
970 let marker = "<|open|>";
971 let marker_id = 256;
972 let mut special_tokens = FxHashMap::default();
973 special_tokens.insert(marker.to_string(), marker_id);
974 let tokenizer =
975 TikTokenTokenizer::from_file(&file_path, r"(?s).+", special_tokens).unwrap();
976
977 let structural = tokenizer
978 .encode_segments(&[EncodeSegment::control(marker)])
979 .unwrap();
980 let user_text = tokenizer
981 .encode_segments(&[EncodeSegment::ordinary(marker)])
982 .unwrap();
983
984 assert_eq!(structural.token_ids(), &[marker_id]);
985 assert_ne!(user_text.token_ids(), &[marker_id]);
986 let decoded: String = tokenizer
987 .decode(user_text.token_ids(), false)
988 .unwrap()
989 .into();
990 assert_eq!(decoded, marker);
991 }
992
993 fn piece_chars(text: &str) -> Vec<usize> {
996 let mut lengths = Vec::new();
997 for_each_reference_piece(text, |piece| lengths.push(piece.chars().count()));
998 lengths
999 }
1000
1001 #[test]
1002 fn test_reference_pieces_match_kimi_reference_split() {
1003 assert_eq!(piece_chars("hello world"), vec![11]);
1006 assert_eq!(
1007 piece_chars(&"a".repeat(60_000)),
1008 vec![25_000, 25_000, 10_000]
1009 );
1010 assert_eq!(
1011 piece_chars(&("a".repeat(30_000) + &" ".repeat(30_000))),
1012 vec![25_000, 30_000, 5_000]
1013 );
1014 assert_eq!(
1015 piece_chars(&("x".repeat(25_000) + " " + &"y".repeat(25_001))),
1016 vec![50_001, 1]
1017 );
1018 assert_eq!(
1022 piece_chars(&("a".repeat(20_000) + &"\u{1c}".repeat(10_000))),
1023 vec![30_000]
1024 );
1025 assert_eq!(piece_chars(&"Γ©".repeat(26_000)), vec![25_000, 1_000]);
1027 let chunked: String = "ab ".repeat(136_667).chars().take(410_000).collect();
1029 assert_eq!(piece_chars(&chunked), vec![400_000, 10_000]);
1030 }
1031
1032 #[test]
1033 fn test_segment_encoding_cuts_long_runs_like_the_reference() {
1034 let dir = tempfile::tempdir().unwrap();
1036 let engine = base64::engine::general_purpose::STANDARD;
1037 let mut content = String::new();
1038 for byte_val in 0u16..256 {
1039 let encoded = engine.encode([byte_val as u8]);
1040 content.push_str(&format!("{encoded} {byte_val}\n"));
1041 }
1042 content.push_str(&format!("{} 256\n", engine.encode(b"aa")));
1043 let file_path = dir.path().join("tiktoken.model");
1044 std::fs::write(&file_path, &content).unwrap();
1045 let tokenizer = TikTokenTokenizer::from_file(
1046 file_path.to_str().unwrap(),
1047 r"(?s).+",
1048 FxHashMap::default(),
1049 )
1050 .unwrap();
1051
1052 let text = "b".to_string() + &"a".repeat(25_000);
1055 let segmented = tokenizer
1056 .encode_segments(&[EncodeSegment::ordinary(&text)])
1057 .unwrap();
1058 let mut expected = tokenizer.bpe.encode_ordinary(&text[..25_000]);
1059 expected.extend(tokenizer.bpe.encode_ordinary(&text[25_000..]));
1060 assert_eq!(segmented.token_ids(), expected.as_slice());
1061 assert_ne!(
1062 segmented.token_ids(),
1063 tokenizer.bpe.encode_ordinary(&text).as_slice(),
1064 "the cut must change ids for this input, or the test proves nothing"
1065 );
1066 }
1067
1068 fn has_nontrivial_self_overlap(token: &str) -> bool {
1069 let bytes = token.as_bytes();
1070 (1..bytes.len()).any(|overlap| bytes[bytes.len() - overlap..] == bytes[..overlap])
1071 }
1072
1073 fn have_ambiguous_overlap(a: &str, b: &str) -> bool {
1074 let a = a.as_bytes();
1075 let b = b.as_bytes();
1076
1077 if a.windows(b.len()).any(|window| window == b)
1078 || b.windows(a.len()).any(|window| window == a)
1079 {
1080 return true;
1081 }
1082
1083 let max_overlap = a.len().min(b.len());
1084 (1..max_overlap).any(|overlap| {
1085 a[a.len() - overlap..] == b[..overlap] || b[b.len() - overlap..] == a[..overlap]
1086 })
1087 }
1088
1089 fn assert_special_split_invariant(
1090 tokenizer: &TikTokenTokenizer,
1091 left: &str,
1092 special: &str,
1093 right: &str,
1094 ) {
1095 let whole = tokenizer
1096 .encode(&format!("{left}{special}{right}"))
1097 .unwrap()
1098 .token_ids()
1099 .to_vec();
1100 let mut split = tokenizer
1101 .encode(&format!("{left}{special}"))
1102 .unwrap()
1103 .token_ids()
1104 .to_vec();
1105 split.extend_from_slice(tokenizer.encode(right).unwrap().token_ids());
1106 assert_eq!(
1107 whole, split,
1108 "splitting after registered special token {special:?} must preserve tokenization"
1109 );
1110 }
1111
1112 #[test]
1113 fn test_registered_special_tokens_are_cache_safe_boundaries() {
1114 let dir = tempfile::tempdir().unwrap();
1115 let file_path = create_byte_level_tiktoken_file(dir.path());
1116 create_test_config(dir.path(), "kimi");
1117 create_test_tokenizer_config(dir.path(), 256);
1118
1119 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1120 let specials = tokenizer.special_tokens();
1121 assert!(!specials.is_empty());
1122
1123 for (index, special) in specials.iter().enumerate() {
1124 assert!(
1125 !has_nontrivial_self_overlap(special),
1126 "registered special token has an ambiguous self-overlap: {special:?}"
1127 );
1128 for other in &specials[index + 1..] {
1129 assert!(
1130 !have_ambiguous_overlap(special, other),
1131 "registered special tokens overlap ambiguously: {special:?} and {other:?}"
1132 );
1133 }
1134
1135 for (left, right) in [
1136 ("ordinary prefix ", " ordinary suffix"),
1137 ("line before\n", "\tUnicode after: εδΊ¬ π"),
1138 ("", " trailing text"),
1139 ] {
1140 assert_special_split_invariant(&tokenizer, left, special, right);
1141 }
1142 }
1143
1144 let bos = specials
1145 .iter()
1146 .find(|token| token.as_str() == "[BOS]")
1147 .unwrap();
1148 let eos = specials
1149 .iter()
1150 .find(|token| token.as_str() == "[EOS]")
1151 .unwrap();
1152 assert_special_split_invariant(
1153 &tokenizer,
1154 "prefix ",
1155 bos,
1156 &format!("{eos}<|reserved_token_258|>Unicode ε°Ύι¨"),
1157 );
1158 }
1159
1160 #[test]
1171 fn test_reserved_token_absolute_id_naming_kimi_k25_regression() {
1172 let dir = tempfile::tempdir().unwrap();
1173 let file_path = create_byte_level_tiktoken_file(dir.path());
1174
1175 create_test_config(dir.path(), "kimi");
1177
1178 create_test_tokenizer_config(dir.path(), 256);
1180
1181 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1182
1183 let single = "<|reserved_token_258|>";
1190 let enc = tokenizer.encode(single).unwrap();
1191 assert_eq!(
1192 enc.token_ids().len(),
1193 1,
1194 "'{single}' should be 1 special token, got {} tokens: {:?}. \
1195 This means fallback naming still uses relative offsets instead of absolute IDs.",
1196 enc.token_ids().len(),
1197 enc.token_ids()
1198 );
1199 assert_eq!(enc.token_ids()[0], 258);
1200
1201 let multi: String = (258u32..268)
1204 .map(|id| format!("<|reserved_token_{id}|>"))
1205 .collect();
1206 let enc_multi = tokenizer.encode(&multi).unwrap();
1207 assert_eq!(
1208 enc_multi.token_ids().len(),
1209 10,
1210 "10 reserved token strings should produce exactly 10 tokens, got {}: {:?}",
1211 enc_multi.token_ids().len(),
1212 enc_multi.token_ids()
1213 );
1214 let expected_ids: Vec<u32> = (258..268).collect();
1215 assert_eq!(enc_multi.token_ids(), &expected_ids);
1216 }
1217
1218 #[test]
1222 fn test_relative_offset_naming_causes_inflation() {
1223 let dir = tempfile::tempdir().unwrap();
1224 let file_path = create_byte_level_tiktoken_file(dir.path());
1225
1226 let _encoder = parse_tiktoken_file(&file_path).unwrap();
1227 let num_base_tokens = 256usize;
1228
1229 let mut bad_special_tokens: FxHashMap<String, u32> = FxHashMap::default();
1231 bad_special_tokens.insert("[BOS]".to_string(), 256);
1232 bad_special_tokens.insert("[EOS]".to_string(), 257);
1233 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
1234 let id = num_base_tokens as u32 + i;
1235 if id != 256 && id != 257 {
1236 bad_special_tokens.insert(format!("<|reserved_token_{i}|>"), id);
1238 }
1239 }
1240
1241 let bad_tokenizer =
1242 TikTokenTokenizer::from_file(&file_path, KIMI_PATTERN, bad_special_tokens).unwrap();
1243
1244 let input = "<|reserved_token_258|>";
1247 let enc = bad_tokenizer.encode(input).unwrap();
1248 assert!(
1249 enc.token_ids().len() > 1,
1250 "With buggy relative-offset naming, '{}' should NOT be recognized as a \
1251 single special token. Got {} token(s): {:?}",
1252 input,
1253 enc.token_ids().len(),
1254 enc.token_ids()
1255 );
1256
1257 let multi: String = (258u32..268)
1259 .map(|id| format!("<|reserved_token_{id}|>"))
1260 .collect();
1261 let enc_multi = bad_tokenizer.encode(&multi).unwrap();
1262 assert!(
1263 enc_multi.token_ids().len() > 10,
1264 "With buggy naming, 10 reserved token strings should inflate beyond 10 tokens. \
1265 Got {}",
1266 enc_multi.token_ids().len(),
1267 );
1268 }
1269}