1use std::collections::HashSet;
5use std::path::Path;
6
7use base64::Engine as _;
8use rayon::prelude::*;
9use rustc_hash::FxHashMap;
10use tiktoken_rs::CoreBPE;
11
12use super::{
13 EncodeSegment, Encoding, Error, Result, TokenIdType,
14 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17const DEFAULT_NUM_RESERVED_SPECIAL_TOKENS: u32 = 256;
20
21const KIMI_PATTERN: &str = fastokens::tiktoken::KIMI_PATTERN;
23
24pub struct TikTokenTokenizer {
25 bpe: CoreBPE,
26 special_token_ids: HashSet<u32>,
27 special_tokens: Vec<String>,
28}
29
30pub(crate) fn sorted_special_token_strings(special_tokens: &FxHashMap<String, u32>) -> Vec<String> {
31 let mut strings: Vec<String> = special_tokens.keys().cloned().collect();
32 strings.sort();
33 strings.dedup();
34 strings
35}
36
37impl TikTokenTokenizer {
38 pub fn from_file(
45 path: &str,
46 pattern: &str,
47 special_tokens: FxHashMap<String, u32>,
48 ) -> Result<Self> {
49 let encoder = parse_tiktoken_file(path)?;
50 let special_token_ids: HashSet<u32> = special_tokens.values().copied().collect();
51 let special_token_strings = sorted_special_token_strings(&special_tokens);
52
53 let bpe = CoreBPE::new(encoder, special_tokens, pattern)
54 .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
55
56 Ok(Self {
57 bpe,
58 special_token_ids,
59 special_tokens: special_token_strings,
60 })
61 }
62
63 pub fn from_file_auto(path: &str) -> Result<Self> {
68 let file_path = Path::new(path);
69 let directory = file_path
70 .parent()
71 .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
72
73 let pattern = detect_bpe_pattern(directory)?;
74 let encoder = parse_tiktoken_file(path)?;
75 let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
77 let AddedTokens { tokens, skip_ids } = load_added_tokens(directory, num_base_tokens)?;
78 let special_token_strings = sorted_special_token_strings(&tokens);
79
80 let bpe = CoreBPE::new(encoder, tokens, pattern)
81 .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
82
83 Ok(Self {
84 bpe,
85 special_token_ids: skip_ids,
86 special_tokens: special_token_strings,
87 })
88 }
89
90 pub fn special_tokens(&self) -> &[String] {
97 &self.special_tokens
98 }
99}
100
101impl Encoder for TikTokenTokenizer {
102 fn encode(&self, input: &str) -> Result<Encoding> {
103 let token_ids: Vec<u32> = self.bpe.encode_with_special_tokens(input);
104 Ok(Encoding::Sp(token_ids))
105 }
106
107 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
108 inputs.par_iter().map(|input| self.encode(input)).collect()
109 }
110
111 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
112 let mut token_ids = Vec::new();
113 for segment in segments {
114 for_each_reference_piece(segment.text, |piece| {
115 if segment.allow_special {
116 token_ids.extend(self.bpe.encode_with_special_tokens(piece));
117 } else {
118 token_ids.extend(self.bpe.encode_ordinary(piece));
119 }
120 });
121 }
122 Ok(Encoding::Sp(token_ids))
123 }
124}
125
126const MAX_ENCODE_CHARS: usize = 400_000;
128
129const MAX_RUN_CHARS: usize = 25_000;
132
133fn is_python_space(c: char) -> bool {
136 c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
137}
138
139fn for_each_reference_piece<'a>(text: &'a str, mut f: impl FnMut(&'a str)) {
146 if text.len() <= MAX_RUN_CHARS {
148 f(text);
149 return;
150 }
151 let mut chunk_start = 0;
152 let mut chunk_chars = 0;
153 for (index, _) in text.char_indices() {
154 if chunk_chars == MAX_ENCODE_CHARS {
155 split_long_runs(&text[chunk_start..index], &mut f);
156 chunk_start = index;
157 chunk_chars = 0;
158 }
159 chunk_chars += 1;
160 }
161 split_long_runs(&text[chunk_start..], &mut f);
162}
163
164fn split_long_runs<'a>(chunk: &'a str, f: &mut impl FnMut(&'a str)) {
167 let mut run_is_space = chunk.chars().next().is_some_and(is_python_space);
168 let mut run_len = 0;
169 let mut piece_start = 0;
170 for (index, c) in chunk.char_indices() {
171 let is_space = is_python_space(c);
172 if is_space != run_is_space {
173 run_is_space = is_space;
174 run_len = 1;
175 } else {
176 run_len += 1;
177 if run_len > MAX_RUN_CHARS {
178 f(&chunk[piece_start..index]);
179 piece_start = index;
180 run_len = 1;
181 }
182 }
183 }
184 f(&chunk[piece_start..]);
185}
186
187impl Decoder for TikTokenTokenizer {
188 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
189 let ids: Vec<u32> = if skip_special_tokens {
190 token_ids
191 .iter()
192 .filter(|&&id| !self.special_token_ids.contains(&id))
193 .copied()
194 .collect()
195 } else {
196 token_ids.to_vec()
197 };
198
199 let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
208 match String::from_utf8(bytes) {
209 Ok(text) => Ok(DecodeResult::Complete(text)),
210 Err(e) => {
211 let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
212 Ok(DecodeResult::from_decoded(text))
213 }
214 }
215 }
216}
217
218impl Tokenizer for TikTokenTokenizer {
219 fn validate_prefix_cache(&self) -> Result<()> {
220 Ok(())
221 }
222
223 fn vocab_size(&self) -> Option<usize> {
228 None
229 }
230
231 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
232 let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
233 ids.sort_unstable();
234 Ok(ids)
235 }
236
237 fn num_special_tokens_added(&self) -> Result<usize> {
241 Ok(0)
242 }
243}
244
245pub(crate) fn parse_tiktoken_file(path: &str) -> Result<FxHashMap<Vec<u8>, u32>> {
247 let contents = std::fs::read_to_string(path)
248 .map_err(|err| Error::msg(format!("Failed to read tiktoken file '{path}': {err}")))?;
249
250 let engine = base64::engine::general_purpose::STANDARD;
251 let mut encoder = FxHashMap::default();
252
253 for line in contents.lines() {
254 let line = line.trim();
255 if line.is_empty() {
256 continue;
257 }
258 let mut parts = line.split_whitespace();
259 let token_b64 = parts
260 .next()
261 .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no token): {line}")))?;
262 let rank_str = parts
263 .next()
264 .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no rank): {line}")))?;
265
266 let token_bytes = engine
267 .decode(token_b64)
268 .map_err(|err| Error::msg(format!("Invalid base64 in tiktoken file: {err}")))?;
269 let rank: u32 = rank_str
270 .parse()
271 .map_err(|err| Error::msg(format!("Invalid rank in tiktoken file: {err}")))?;
272
273 encoder.insert(token_bytes, rank);
274 }
275
276 Ok(encoder)
277}
278
279pub(crate) fn detect_bpe_pattern(directory: &Path) -> Result<&'static str> {
281 let model_type: String = crate::file_json_field(&directory.join("config.json"), "model_type")
282 .map_err(|err| {
283 Error::msg(format!("Failed to read model_type from config.json: {err}"))
284 })?;
285
286 match model_type.as_str() {
287 "kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "kimi_k31" | "kimi_linear"
293 | "deepseek_v3" => Ok(KIMI_PATTERN),
294 _ => Err(Error::msg(format!(
295 "Unsupported tiktoken model_type '{model_type}'. \
296 Currently supported: kimi, kimi_k2, kimi_k25, kimi_k3, kimi_k31, kimi_linear, deepseek_v3. \
297 To add a new model type, extend detect_bpe_pattern() in lib/tokenizers/src/tiktoken.rs \
298 with the appropriate BPE regex pattern. \
299 Alternatively, provide a tokenizer.json (HuggingFace format) instead."
300 ))),
301 }
302}
303
304pub(crate) struct AddedTokens {
306 pub(crate) tokens: FxHashMap<String, u32>,
308 pub(crate) skip_ids: HashSet<u32>,
311}
312
313pub(crate) fn load_added_tokens(directory: &Path, num_base_tokens: usize) -> Result<AddedTokens> {
318 let config_path = directory.join("tokenizer_config.json");
319 let mut tokens = FxHashMap::default();
320 let mut skip_ids = HashSet::new();
321
322 if !config_path.exists() {
323 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
325 let id = num_base_tokens as u32 + i;
326 tokens.insert(format!("<|reserved_token_{id}|>"), id);
327 skip_ids.insert(id);
328 }
329 return Ok(AddedTokens { tokens, skip_ids });
330 }
331
332 let contents = std::fs::read_to_string(&config_path)
333 .map_err(|err| Error::msg(format!("Failed to read tokenizer_config.json: {err}")))?;
334
335 let config: serde_json::Value = serde_json::from_str(&contents)
336 .map_err(|err| Error::msg(format!("Failed to parse tokenizer_config.json: {err}")))?;
337
338 if let Some(added_tokens) = config
339 .get("added_tokens_decoder")
340 .and_then(|v| v.as_object())
341 {
342 for (id_str, token_def) in added_tokens {
343 let id: u32 = id_str.parse().map_err(|err| {
344 Error::msg(format!(
345 "Invalid token ID '{id_str}' in added_tokens_decoder: {err}"
346 ))
347 })?;
348
349 let content = token_def
350 .get("content")
351 .and_then(|v| v.as_str())
352 .unwrap_or_else(|| {
353 tracing::warn!("Missing 'content' field for token ID {id}");
355 ""
356 });
357
358 if !content.is_empty() {
359 tokens.insert(content.to_string(), id);
360 if token_def
361 .get("special")
362 .and_then(serde_json::Value::as_bool)
363 == Some(true)
364 {
365 skip_ids.insert(id);
366 }
367 }
368 }
369
370 let used_ids: HashSet<u32> = tokens.values().copied().collect();
372 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
373 let id = num_base_tokens as u32 + i;
374 if !used_ids.contains(&id) {
375 tokens.insert(format!("<|reserved_token_{id}|>"), id);
376 skip_ids.insert(id);
377 }
378 }
379 } else {
380 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
382 let id = num_base_tokens as u32 + i;
383 tokens.insert(format!("<|reserved_token_{id}|>"), id);
384 skip_ids.insert(id);
385 }
386 }
387
388 Ok(AddedTokens { tokens, skip_ids })
389}
390
391#[cfg(test)]
392mod tests {
393 use super::*;
394 use crate::DecodeStream;
395 use std::io::Write;
396 use std::sync::Arc;
397
398 fn create_test_tiktoken_file(dir: &Path) -> String {
399 let engine = base64::engine::general_purpose::STANDARD;
400 let mut content = String::new();
401
402 let tokens: Vec<(&[u8], u32)> = vec![
404 (b"h", 0),
405 (b"e", 1),
406 (b"l", 2),
407 (b"o", 3),
408 (b" ", 4),
409 (b"w", 5),
410 (b"r", 6),
411 (b"d", 7),
412 (b"he", 8),
413 (b"ll", 9),
414 (b"lo", 10),
415 (b"wo", 11),
416 (b"rl", 12),
417 (b"hel", 13),
418 (b"llo", 14),
419 (b"wor", 15),
420 (b"hell", 16),
421 (b"ello", 17),
422 (b"worl", 18),
423 (b"hello", 19),
424 (b"world", 20),
425 ];
426
427 for (token, rank) in tokens {
428 let encoded = engine.encode(token);
429 content.push_str(&format!("{encoded} {rank}\n"));
430 }
431
432 let file_path = dir.join("tiktoken.model");
433 let mut file = std::fs::File::create(&file_path).unwrap();
434 file.write_all(content.as_bytes()).unwrap();
435 file_path.to_str().unwrap().to_string()
436 }
437
438 fn create_test_config(dir: &Path, model_type: &str) {
439 let config = serde_json::json!({
440 "model_type": model_type,
441 "max_position_embeddings": 32768,
442 "eos_token_id": [21]
443 });
444 let file_path = dir.join("config.json");
445 std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
446 }
447
448 fn create_test_tokenizer_config(dir: &Path, num_base_tokens: usize) {
449 let mut added_tokens = serde_json::Map::new();
450 let bos_id = num_base_tokens;
451 let eos_id = num_base_tokens + 1;
452
453 added_tokens.insert(
454 bos_id.to_string(),
455 serde_json::json!({"content": "[BOS]", "special": true}),
456 );
457 added_tokens.insert(
458 eos_id.to_string(),
459 serde_json::json!({"content": "[EOS]", "special": true}),
460 );
461
462 let config = serde_json::json!({
463 "added_tokens_decoder": added_tokens
464 });
465
466 let file_path = dir.join("tokenizer_config.json");
467 std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
468 }
469
470 #[test]
471 fn test_parse_tiktoken_file() {
472 let dir = tempfile::tempdir().unwrap();
473 let file_path = create_test_tiktoken_file(dir.path());
474 let encoder = parse_tiktoken_file(&file_path).unwrap();
475 assert_eq!(encoder.len(), 21);
476 assert_eq!(encoder[b"hello".as_slice()], 19);
477 assert_eq!(encoder[b"world".as_slice()], 20);
478 }
479
480 #[test]
481 fn test_parse_tiktoken_file_missing() {
482 let result = parse_tiktoken_file("/nonexistent/path/tiktoken.model");
483 assert!(result.is_err());
484 }
485
486 #[test]
487 fn test_tiktoken_from_file() {
488 let dir = tempfile::tempdir().unwrap();
489 let file_path = create_test_tiktoken_file(dir.path());
490
491 let mut special_tokens = FxHashMap::default();
492 special_tokens.insert("[BOS]".to_string(), 21_u32);
493 special_tokens.insert("[EOS]".to_string(), 22_u32);
494
495 let pattern = r"[\w]+|[^\w\s]+|\s+";
497
498 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
499
500 assert_eq!(
501 tokenizer.special_tokens(),
502 &["[BOS]".to_string(), "[EOS]".to_string()]
503 );
504
505 let encoding = tokenizer.encode("hello world").unwrap();
507 let ids = encoding.token_ids();
508 assert!(!ids.is_empty());
509
510 let decoded: String = tokenizer.decode(ids, false).unwrap().into();
512 assert_eq!(decoded, "hello world");
513 }
514
515 #[test]
516 fn test_tiktoken_encoding_variant() {
517 let dir = tempfile::tempdir().unwrap();
518 let file_path = create_test_tiktoken_file(dir.path());
519
520 let special_tokens = FxHashMap::default();
521 let pattern = r"[\w]+|[^\w\s]+|\s+";
522
523 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
524 assert!(tokenizer.special_tokens().is_empty());
525 let encoding = tokenizer.encode("hello").unwrap();
526
527 match &encoding {
529 Encoding::Sp(_) => {}
530 other => panic!("Expected Encoding::Sp, got {:?}", other),
531 }
532 }
533
534 #[test]
535 fn test_tiktoken_skip_special_tokens() {
536 let dir = tempfile::tempdir().unwrap();
537 let file_path = create_test_tiktoken_file(dir.path());
538
539 let mut special_tokens = FxHashMap::default();
540 special_tokens.insert("[BOS]".to_string(), 21_u32);
541 special_tokens.insert("[EOS]".to_string(), 22_u32);
542
543 let pattern = r"[\w]+|[^\w\s]+|\s+";
544
545 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
546
547 let encoding = tokenizer.encode("hello").unwrap();
549 let mut ids = vec![21u32]; ids.extend(encoding.token_ids());
551 ids.push(22); let decoded_skip: String = tokenizer.decode(&ids, true).unwrap().into();
555 assert_eq!(decoded_skip, "hello");
556
557 let decoded_all: String = tokenizer.decode(&ids, false).unwrap().into();
559 assert!(decoded_all.contains("hello"));
560 }
561
562 #[test]
563 fn test_special_token_ids_reports_configured_ids_sorted() {
564 let dir = tempfile::tempdir().unwrap();
565 let file_path = create_test_tiktoken_file(dir.path());
566
567 let mut special_tokens = FxHashMap::default();
568 special_tokens.insert("[EOS]".to_string(), 22_u32);
569 special_tokens.insert("[BOS]".to_string(), 21_u32);
570
571 let pattern = r"[\w]+|[^\w\s]+|\s+";
572 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
573
574 assert_eq!(tokenizer.special_token_ids().unwrap(), vec![21, 22]);
575 }
576
577 #[test]
578 fn test_vocab_size_and_token_to_id_are_unsupported() {
579 let dir = tempfile::tempdir().unwrap();
580 let file_path = create_test_tiktoken_file(dir.path());
581 let pattern = r"[\w]+|[^\w\s]+|\s+";
582 let tokenizer =
583 TikTokenTokenizer::from_file(&file_path, pattern, FxHashMap::default()).unwrap();
584
585 assert_eq!(tokenizer.vocab_size(), None);
586 assert!(tokenizer.token_to_id("hello").is_err());
587 }
588
589 #[test]
590 fn test_tiktoken_from_file_auto() {
591 let dir = tempfile::tempdir().unwrap();
592 let file_path = create_test_tiktoken_file(dir.path());
593
594 create_test_config(dir.path(), "kimi");
595 create_test_tokenizer_config(dir.path(), 21);
596
597 let mut expected_specials: Vec<String> = load_added_tokens(dir.path(), 21)
598 .unwrap()
599 .tokens
600 .into_keys()
601 .collect();
602 expected_specials.sort();
603 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
604 assert_eq!(tokenizer.special_tokens(), expected_specials);
605
606 let encoding = tokenizer.encode("hello world").unwrap();
608 let ids = encoding.token_ids();
609 assert!(!ids.is_empty());
610
611 let decoded: String = tokenizer.decode(ids, false).unwrap().into();
612 assert_eq!(decoded, "hello world");
613
614 create_test_config(dir.path(), "kimi_k31");
616 let k31_tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
617 assert_eq!(
618 k31_tokenizer.encode("hello world").unwrap().token_ids(),
619 ids
620 );
621 let decoded: String = k31_tokenizer.decode(ids, false).unwrap().into();
622 assert_eq!(decoded, "hello world");
623 }
624
625 #[test]
626 fn test_detect_bpe_pattern_unknown() {
627 let dir = tempfile::tempdir().unwrap();
628 create_test_config(dir.path(), "unknown_model");
629 let result = detect_bpe_pattern(dir.path());
630 assert!(result.is_err());
631 }
632
633 #[test]
634 fn test_kimi_k3_uses_kimi_bpe_pattern() {
635 let dir = tempfile::tempdir().unwrap();
636 for model_type in ["kimi_k3", "kimi_k31"] {
637 create_test_config(dir.path(), model_type);
638 assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
639 }
640 }
641
642 #[test]
643 fn test_detect_bpe_pattern_kimi_linear() {
644 let dir = tempfile::tempdir().unwrap();
645 create_test_config(dir.path(), "kimi_linear");
646 assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
647 }
648
649 #[test]
650 fn test_load_special_tokens_no_config() {
651 let dir = tempfile::tempdir().unwrap();
652 let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
653 assert_eq!(tokens.len(), 256);
654 assert_eq!(tokens["<|reserved_token_100|>"], 100);
655 assert_eq!(tokens["<|reserved_token_355|>"], 355);
656 }
657
658 #[test]
659 fn test_load_special_tokens_with_config() {
660 let dir = tempfile::tempdir().unwrap();
661 create_test_tokenizer_config(dir.path(), 100);
662 let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
663 assert_eq!(tokens["[BOS]"], 100);
664 assert_eq!(tokens["[EOS]"], 101);
665 assert!(tokens.len() > 2);
667 }
668
669 fn create_tokenizer_with_non_special_added_tokens(dir: &Path) -> TikTokenTokenizer {
672 let file_path = create_test_tiktoken_file(dir);
673 create_test_config(dir, "kimi_k3");
674 let config = serde_json::json!({"added_tokens_decoder": {
675 "21": {"content": "[BOS]", "special": true},
676 "22": {"content": "<|open|>", "special": false},
677 "23": {"content": "<|sep|>"},
678 }});
679 std::fs::write(dir.join("tokenizer_config.json"), config.to_string()).unwrap();
680 TikTokenTokenizer::from_file_auto(&file_path).unwrap()
681 }
682
683 #[test]
684 fn test_skip_special_tokens_keeps_non_special_added_tokens() {
685 let dir = tempfile::tempdir().unwrap();
686 let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
687
688 let encoding = tokenizer.encode("<|open|>hello").unwrap();
689 assert_eq!(encoding.token_ids(), &[22, 19]);
690
691 let special_ids = tokenizer.special_token_ids().unwrap();
692 assert!(special_ids.contains(&21));
693 assert!(special_ids.contains(&24), "reserved placeholder");
694 assert!(!special_ids.contains(&22));
695 assert!(!special_ids.contains(&23));
696
697 let ids = [21, 22, 19, 23];
698 let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
699 assert_eq!(skipped, "<|open|>hello<|sep|>");
700 let kept: String = tokenizer.decode(&ids, false).unwrap().into();
701 assert_eq!(kept, "[BOS]<|open|>hello<|sep|>");
702 }
703
704 #[test]
705 fn test_decode_stream_emits_leading_non_special_added_token() {
706 let dir = tempfile::tempdir().unwrap();
707 let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
708 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
709
710 let mut stream = DecodeStream::new(tokenizer_arc, &[19], true);
711 assert_eq!(stream.step(22).unwrap().as_deref(), Some("<|open|>"));
712 assert_eq!(stream.step(19).unwrap().as_deref(), Some("hello"));
713 assert_eq!(stream.step(21).unwrap(), None);
714 assert_eq!(stream.step(23).unwrap().as_deref(), Some("<|sep|>"));
715 }
716
717 fn create_test_tiktoken_file_with_byte_tokens(dir: &Path) -> String {
719 let engine = base64::engine::general_purpose::STANDARD;
720 let mut content = String::new();
721
722 let tokens: Vec<(&[u8], u32)> = vec![
723 (b"h", 0),
724 (b"e", 1),
725 (b"l", 2),
726 (b"o", 3),
727 (b" ", 4),
728 (b"hello", 5),
729 ];
730
731 for (token, rank) in &tokens {
732 let encoded = engine.encode(token);
733 content.push_str(&format!("{encoded} {rank}\n"));
734 }
735
736 let byte_tokens: Vec<(Vec<u8>, u32)> =
739 vec![(vec![0xE4], 100), (vec![0xBD], 101), (vec![0xA0], 102)];
740
741 for (token, rank) in &byte_tokens {
742 let encoded = engine.encode(token);
743 content.push_str(&format!("{encoded} {rank}\n"));
744 }
745
746 let emoji_tokens: Vec<(Vec<u8>, u32)> = vec![
748 (vec![0xF0], 200),
749 (vec![0x9F], 201),
750 (vec![0x98], 202),
751 (vec![0x80], 203),
752 ];
753
754 for (token, rank) in &emoji_tokens {
755 let encoded = engine.encode(token);
756 content.push_str(&format!("{encoded} {rank}\n"));
757 }
758
759 let fffd_token: Vec<(Vec<u8>, u32)> = vec![(vec![0xEF, 0xBF, 0xBD], 300)];
762
763 for (token, rank) in &fffd_token {
764 let encoded = engine.encode(token);
765 content.push_str(&format!("{encoded} {rank}\n"));
766 }
767
768 let file_path = dir.join("tiktoken.model");
769 let mut file = std::fs::File::create(&file_path).unwrap();
770 file.write_all(content.as_bytes()).unwrap();
771 file_path.to_str().unwrap().to_string()
772 }
773
774 fn create_byte_token_tokenizer(dir: &Path) -> TikTokenTokenizer {
775 let file_path = create_test_tiktoken_file_with_byte_tokens(dir);
776 let special_tokens = FxHashMap::default();
777 let pattern = r"[\w]+|[^\w\s]+|\s+";
778 TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap()
779 }
780
781 #[test]
785 fn test_decode_single_incomplete_utf8_byte_does_not_error() {
786 let dir = tempfile::tempdir().unwrap();
787 let tokenizer = create_byte_token_tokenizer(dir.path());
788
789 let result = tokenizer.decode(&[100], false);
790 assert!(
791 result.is_ok(),
792 "decode() should not error on incomplete UTF-8 bytes"
793 );
794 let decode_result = result.unwrap();
795 assert!(
796 decode_result.is_partial(),
797 "incomplete UTF-8 byte should produce DecodeResult::Partial, got: {:?}",
798 decode_result
799 );
800 }
801
802 #[test]
804 fn test_decode_two_of_three_utf8_bytes_does_not_error() {
805 let dir = tempfile::tempdir().unwrap();
806 let tokenizer = create_byte_token_tokenizer(dir.path());
807
808 let result = tokenizer.decode(&[100, 101], false);
809 assert!(result.is_ok());
810 let decode_result = result.unwrap();
811 assert!(
812 decode_result.is_partial(),
813 "incomplete 2-of-3 UTF-8 bytes should produce DecodeResult::Partial, got: {:?}",
814 decode_result
815 );
816 }
817
818 #[test]
822 fn test_decode_complete_multibyte_utf8_produces_correct_char() {
823 let dir = tempfile::tempdir().unwrap();
824 let tokenizer = create_byte_token_tokenizer(dir.path());
825
826 let result = tokenizer.decode(&[100, 101, 102], false);
827 assert!(result.is_ok());
828 assert_eq!(String::from(result.unwrap()), "δ½ ");
829 }
830
831 #[test]
834 fn test_decode_complete_4byte_emoji_from_byte_tokens() {
835 let dir = tempfile::tempdir().unwrap();
836 let tokenizer = create_byte_token_tokenizer(dir.path());
837
838 let result = tokenizer.decode(&[200, 201, 202, 203], false);
839 assert!(result.is_ok());
840 assert_eq!(String::from(result.unwrap()), "π");
841 }
842
843 #[test]
848 fn test_decode_legitimate_replacement_char_token_is_complete() {
849 let dir = tempfile::tempdir().unwrap();
850 let tokenizer = create_byte_token_tokenizer(dir.path());
851
852 let result = tokenizer.decode(&[300], false);
853 assert!(result.is_ok());
854 let decode_result = result.unwrap();
855 assert!(
856 decode_result.is_complete(),
857 "legitimate U+FFFD vocab token must be Complete, got: {:?}",
858 decode_result
859 );
860 assert_eq!(decode_result.as_str(), "\u{FFFD}");
861 }
862
863 #[test]
865 fn test_decode_partial_emoji_does_not_error() {
866 let dir = tempfile::tempdir().unwrap();
867 let tokenizer = create_byte_token_tokenizer(dir.path());
868
869 let result = tokenizer.decode(&[200], false);
870 assert!(result.is_ok());
871 assert!(result.unwrap().is_partial());
872 }
873
874 #[test]
876 fn test_decode_mixed_ascii_and_incomplete_bytes() {
877 let dir = tempfile::tempdir().unwrap();
878 let tokenizer = create_byte_token_tokenizer(dir.path());
879
880 let result = tokenizer.decode(&[5, 100], false);
881 assert!(result.is_ok());
882 let decode_result = result.unwrap();
883 assert!(
884 decode_result.is_partial(),
885 "trailing incomplete byte should produce DecodeResult::Partial"
886 );
887 let text: String = decode_result.into();
888 assert!(
889 text.starts_with("hello"),
890 "should start with 'hello', got: {:?}",
891 text
892 );
893 }
894
895 #[test]
899 fn test_decode_stream_incremental_multibyte_reassembly() {
900 let dir = tempfile::tempdir().unwrap();
901 let tokenizer = create_byte_token_tokenizer(dir.path());
902 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
903
904 let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
905
906 let r1 = stream.step(100).unwrap();
907 assert_eq!(r1, None, "first byte of 3-byte char should be buffered");
908
909 let r2 = stream.step(101).unwrap();
910 assert_eq!(r2, None, "second byte of 3-byte char should be buffered");
911
912 let r3 = stream.step(102).unwrap();
913 assert!(r3.is_some(), "third byte should complete the character");
914 assert_eq!(r3.unwrap(), "δ½ ");
915 }
916
917 #[test]
919 fn test_decode_stream_incremental_emoji_reassembly() {
920 let dir = tempfile::tempdir().unwrap();
921 let tokenizer = create_byte_token_tokenizer(dir.path());
922 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
923
924 let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
925
926 let r1 = stream.step(200).unwrap();
927 assert_eq!(r1, None, "byte 1/4 of emoji should be buffered");
928
929 let r2 = stream.step(201).unwrap();
930 assert_eq!(r2, None, "byte 2/4 of emoji should be buffered");
931
932 let r3 = stream.step(202).unwrap();
933 assert_eq!(r3, None, "byte 3/4 of emoji should be buffered");
934
935 let r4 = stream.step(203).unwrap();
936 assert!(r4.is_some(), "byte 4/4 should complete the emoji");
937 assert_eq!(r4.unwrap(), "π");
938 }
939
940 #[test]
941 fn test_tiktoken_encode_batch() {
942 let dir = tempfile::tempdir().unwrap();
943 let file_path = create_test_tiktoken_file(dir.path());
944
945 let special_tokens = FxHashMap::default();
946 let pattern = r"[\w]+|[^\w\s]+|\s+";
947
948 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
949
950 let inputs = &["hello", "world"];
951 let encodings = tokenizer.encode_batch(inputs).unwrap();
952 assert_eq!(encodings.len(), 2);
953
954 for (encoding, input) in encodings.iter().zip(inputs.iter()) {
955 let decoded: String = tokenizer
956 .decode(encoding.token_ids(), false)
957 .unwrap()
958 .into();
959 assert_eq!(decoded, *input);
960 }
961 }
962
963 fn create_byte_level_tiktoken_file(dir: &Path) -> String {
966 let engine = base64::engine::general_purpose::STANDARD;
967 let mut content = String::new();
968 for byte_val in 0u16..256 {
969 let encoded = engine.encode([byte_val as u8]);
970 content.push_str(&format!("{encoded} {byte_val}\n"));
971 }
972 let file_path = dir.join("tiktoken.model");
973 std::fs::write(&file_path, &content).unwrap();
974 file_path.to_str().unwrap().to_string()
975 }
976
977 #[test]
978 fn test_segment_encoding_preserves_special_vs_ordinary_boundary() {
979 let dir = tempfile::tempdir().unwrap();
980 let file_path = create_byte_level_tiktoken_file(dir.path());
981 let marker = "<|open|>";
982 let marker_id = 256;
983 let mut special_tokens = FxHashMap::default();
984 special_tokens.insert(marker.to_string(), marker_id);
985 let tokenizer =
986 TikTokenTokenizer::from_file(&file_path, r"(?s).+", special_tokens).unwrap();
987
988 let structural = tokenizer
989 .encode_segments(&[EncodeSegment::control(marker)])
990 .unwrap();
991 let user_text = tokenizer
992 .encode_segments(&[EncodeSegment::ordinary(marker)])
993 .unwrap();
994
995 assert_eq!(structural.token_ids(), &[marker_id]);
996 assert_ne!(user_text.token_ids(), &[marker_id]);
997 let decoded: String = tokenizer
998 .decode(user_text.token_ids(), false)
999 .unwrap()
1000 .into();
1001 assert_eq!(decoded, marker);
1002 }
1003
1004 fn piece_chars(text: &str) -> Vec<usize> {
1007 let mut lengths = Vec::new();
1008 for_each_reference_piece(text, |piece| lengths.push(piece.chars().count()));
1009 lengths
1010 }
1011
1012 #[test]
1013 fn test_reference_pieces_match_kimi_reference_split() {
1014 assert_eq!(piece_chars("hello world"), vec![11]);
1017 assert_eq!(
1018 piece_chars(&"a".repeat(60_000)),
1019 vec![25_000, 25_000, 10_000]
1020 );
1021 assert_eq!(
1022 piece_chars(&("a".repeat(30_000) + &" ".repeat(30_000))),
1023 vec![25_000, 30_000, 5_000]
1024 );
1025 assert_eq!(
1026 piece_chars(&("x".repeat(25_000) + " " + &"y".repeat(25_001))),
1027 vec![50_001, 1]
1028 );
1029 assert_eq!(
1033 piece_chars(&("a".repeat(20_000) + &"\u{1c}".repeat(10_000))),
1034 vec![30_000]
1035 );
1036 assert_eq!(piece_chars(&"Γ©".repeat(26_000)), vec![25_000, 1_000]);
1038 let chunked: String = "ab ".repeat(136_667).chars().take(410_000).collect();
1040 assert_eq!(piece_chars(&chunked), vec![400_000, 10_000]);
1041 }
1042
1043 #[test]
1044 fn test_segment_encoding_cuts_long_runs_like_the_reference() {
1045 let dir = tempfile::tempdir().unwrap();
1047 let engine = base64::engine::general_purpose::STANDARD;
1048 let mut content = String::new();
1049 for byte_val in 0u16..256 {
1050 let encoded = engine.encode([byte_val as u8]);
1051 content.push_str(&format!("{encoded} {byte_val}\n"));
1052 }
1053 content.push_str(&format!("{} 256\n", engine.encode(b"aa")));
1054 let file_path = dir.path().join("tiktoken.model");
1055 std::fs::write(&file_path, &content).unwrap();
1056 let tokenizer = TikTokenTokenizer::from_file(
1057 file_path.to_str().unwrap(),
1058 r"(?s).+",
1059 FxHashMap::default(),
1060 )
1061 .unwrap();
1062
1063 let text = "b".to_string() + &"a".repeat(25_000);
1066 let segmented = tokenizer
1067 .encode_segments(&[EncodeSegment::ordinary(&text)])
1068 .unwrap();
1069 let mut expected = tokenizer.bpe.encode_ordinary(&text[..25_000]);
1070 expected.extend(tokenizer.bpe.encode_ordinary(&text[25_000..]));
1071 assert_eq!(segmented.token_ids(), expected.as_slice());
1072 assert_ne!(
1073 segmented.token_ids(),
1074 tokenizer.bpe.encode_ordinary(&text).as_slice(),
1075 "the cut must change ids for this input, or the test proves nothing"
1076 );
1077 }
1078
1079 fn has_nontrivial_self_overlap(token: &str) -> bool {
1080 let bytes = token.as_bytes();
1081 (1..bytes.len()).any(|overlap| bytes[bytes.len() - overlap..] == bytes[..overlap])
1082 }
1083
1084 fn have_ambiguous_overlap(a: &str, b: &str) -> bool {
1085 let a = a.as_bytes();
1086 let b = b.as_bytes();
1087
1088 if a.windows(b.len()).any(|window| window == b)
1089 || b.windows(a.len()).any(|window| window == a)
1090 {
1091 return true;
1092 }
1093
1094 let max_overlap = a.len().min(b.len());
1095 (1..max_overlap).any(|overlap| {
1096 a[a.len() - overlap..] == b[..overlap] || b[b.len() - overlap..] == a[..overlap]
1097 })
1098 }
1099
1100 fn assert_special_split_invariant(
1101 tokenizer: &TikTokenTokenizer,
1102 left: &str,
1103 special: &str,
1104 right: &str,
1105 ) {
1106 let whole = tokenizer
1107 .encode(&format!("{left}{special}{right}"))
1108 .unwrap()
1109 .token_ids()
1110 .to_vec();
1111 let mut split = tokenizer
1112 .encode(&format!("{left}{special}"))
1113 .unwrap()
1114 .token_ids()
1115 .to_vec();
1116 split.extend_from_slice(tokenizer.encode(right).unwrap().token_ids());
1117 assert_eq!(
1118 whole, split,
1119 "splitting after registered special token {special:?} must preserve tokenization"
1120 );
1121 }
1122
1123 #[test]
1124 fn test_registered_special_tokens_are_cache_safe_boundaries() {
1125 let dir = tempfile::tempdir().unwrap();
1126 let file_path = create_byte_level_tiktoken_file(dir.path());
1127 create_test_config(dir.path(), "kimi");
1128 create_test_tokenizer_config(dir.path(), 256);
1129
1130 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1131 let specials = tokenizer.special_tokens();
1132 assert!(!specials.is_empty());
1133
1134 for (index, special) in specials.iter().enumerate() {
1135 assert!(
1136 !has_nontrivial_self_overlap(special),
1137 "registered special token has an ambiguous self-overlap: {special:?}"
1138 );
1139 for other in &specials[index + 1..] {
1140 assert!(
1141 !have_ambiguous_overlap(special, other),
1142 "registered special tokens overlap ambiguously: {special:?} and {other:?}"
1143 );
1144 }
1145
1146 for (left, right) in [
1147 ("ordinary prefix ", " ordinary suffix"),
1148 ("line before\n", "\tUnicode after: εδΊ¬ π"),
1149 ("", " trailing text"),
1150 ] {
1151 assert_special_split_invariant(&tokenizer, left, special, right);
1152 }
1153 }
1154
1155 let bos = specials
1156 .iter()
1157 .find(|token| token.as_str() == "[BOS]")
1158 .unwrap();
1159 let eos = specials
1160 .iter()
1161 .find(|token| token.as_str() == "[EOS]")
1162 .unwrap();
1163 assert_special_split_invariant(
1164 &tokenizer,
1165 "prefix ",
1166 bos,
1167 &format!("{eos}<|reserved_token_258|>Unicode ε°Ύι¨"),
1168 );
1169 }
1170
1171 #[test]
1182 fn test_reserved_token_absolute_id_naming_kimi_k25_regression() {
1183 let dir = tempfile::tempdir().unwrap();
1184 let file_path = create_byte_level_tiktoken_file(dir.path());
1185
1186 create_test_config(dir.path(), "kimi");
1188
1189 create_test_tokenizer_config(dir.path(), 256);
1191
1192 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1193
1194 let single = "<|reserved_token_258|>";
1201 let enc = tokenizer.encode(single).unwrap();
1202 assert_eq!(
1203 enc.token_ids().len(),
1204 1,
1205 "'{single}' should be 1 special token, got {} tokens: {:?}. \
1206 This means fallback naming still uses relative offsets instead of absolute IDs.",
1207 enc.token_ids().len(),
1208 enc.token_ids()
1209 );
1210 assert_eq!(enc.token_ids()[0], 258);
1211
1212 let multi: String = (258u32..268)
1215 .map(|id| format!("<|reserved_token_{id}|>"))
1216 .collect();
1217 let enc_multi = tokenizer.encode(&multi).unwrap();
1218 assert_eq!(
1219 enc_multi.token_ids().len(),
1220 10,
1221 "10 reserved token strings should produce exactly 10 tokens, got {}: {:?}",
1222 enc_multi.token_ids().len(),
1223 enc_multi.token_ids()
1224 );
1225 let expected_ids: Vec<u32> = (258..268).collect();
1226 assert_eq!(enc_multi.token_ids(), &expected_ids);
1227 }
1228
1229 #[test]
1233 fn test_relative_offset_naming_causes_inflation() {
1234 let dir = tempfile::tempdir().unwrap();
1235 let file_path = create_byte_level_tiktoken_file(dir.path());
1236
1237 let _encoder = parse_tiktoken_file(&file_path).unwrap();
1238 let num_base_tokens = 256usize;
1239
1240 let mut bad_special_tokens: FxHashMap<String, u32> = FxHashMap::default();
1242 bad_special_tokens.insert("[BOS]".to_string(), 256);
1243 bad_special_tokens.insert("[EOS]".to_string(), 257);
1244 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
1245 let id = num_base_tokens as u32 + i;
1246 if id != 256 && id != 257 {
1247 bad_special_tokens.insert(format!("<|reserved_token_{i}|>"), id);
1249 }
1250 }
1251
1252 let bad_tokenizer =
1253 TikTokenTokenizer::from_file(&file_path, KIMI_PATTERN, bad_special_tokens).unwrap();
1254
1255 let input = "<|reserved_token_258|>";
1258 let enc = bad_tokenizer.encode(input).unwrap();
1259 assert!(
1260 enc.token_ids().len() > 1,
1261 "With buggy relative-offset naming, '{}' should NOT be recognized as a \
1262 single special token. Got {} token(s): {:?}",
1263 input,
1264 enc.token_ids().len(),
1265 enc.token_ids()
1266 );
1267
1268 let multi: String = (258u32..268)
1270 .map(|id| format!("<|reserved_token_{id}|>"))
1271 .collect();
1272 let enc_multi = bad_tokenizer.encode(&multi).unwrap();
1273 assert!(
1274 enc_multi.token_ids().len() > 10,
1275 "With buggy naming, 10 reserved token strings should inflate beyond 10 tokens. \
1276 Got {}",
1277 enc_multi.token_ids().len(),
1278 );
1279 }
1280}