1use std::collections::HashSet;
5use std::path::Path;
6
7use base64::Engine as _;
8use rayon::prelude::*;
9use rustc_hash::FxHashMap;
10use tiktoken_rs::CoreBPE;
11
12use super::{
13 EncodeSegment, Encoding, Error, Result, TokenIdType,
14 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17const DEFAULT_NUM_RESERVED_SPECIAL_TOKENS: u32 = 256;
20
21const KIMI_PATTERN: &str = fastokens::tiktoken::KIMI_PATTERN;
23
24pub struct TikTokenTokenizer {
25 bpe: CoreBPE,
26 special_token_ids: HashSet<u32>,
27 special_tokens: Vec<String>,
28}
29
30pub(crate) fn sorted_special_token_strings(special_tokens: &FxHashMap<String, u32>) -> Vec<String> {
31 let mut strings: Vec<String> = special_tokens.keys().cloned().collect();
32 strings.sort();
33 strings.dedup();
34 strings
35}
36
37impl TikTokenTokenizer {
38 pub fn from_file(
45 path: &str,
46 pattern: &str,
47 special_tokens: FxHashMap<String, u32>,
48 ) -> Result<Self> {
49 let encoder = parse_tiktoken_file(path)?;
50 let special_token_ids: HashSet<u32> = special_tokens.values().copied().collect();
51 let special_token_strings = sorted_special_token_strings(&special_tokens);
52
53 let bpe = CoreBPE::new(encoder, special_tokens, pattern)
54 .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
55
56 Ok(Self {
57 bpe,
58 special_token_ids,
59 special_tokens: special_token_strings,
60 })
61 }
62
63 pub fn from_file_auto(path: &str) -> Result<Self> {
68 let file_path = Path::new(path);
69 let directory = file_path
70 .parent()
71 .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
72
73 let pattern = detect_bpe_pattern(directory)?;
74 let encoder = parse_tiktoken_file(path)?;
75 let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
77 let AddedTokens { tokens, skip_ids } = load_added_tokens(directory, num_base_tokens)?;
78 let special_token_strings = sorted_special_token_strings(&tokens);
79
80 let bpe = CoreBPE::new(encoder, tokens, pattern)
81 .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
82
83 Ok(Self {
84 bpe,
85 special_token_ids: skip_ids,
86 special_tokens: special_token_strings,
87 })
88 }
89
90 pub fn special_tokens(&self) -> &[String] {
97 &self.special_tokens
98 }
99}
100
101impl Encoder for TikTokenTokenizer {
102 fn encode(&self, input: &str) -> Result<Encoding> {
103 let token_ids: Vec<u32> = self.bpe.encode_with_special_tokens(input);
104 Ok(Encoding::Sp(token_ids))
105 }
106
107 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
108 inputs.par_iter().map(|input| self.encode(input)).collect()
109 }
110
111 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
112 let mut token_ids = Vec::new();
113 for segment in segments {
114 let start = token_ids.len();
115 for_each_reference_piece(segment.text, |piece| {
116 if segment.allow_special {
117 token_ids.extend(self.bpe.encode_with_special_tokens(piece));
118 } else {
119 token_ids.extend(self.bpe.encode_ordinary(piece));
120 }
121 });
122 if !segment.allow_special {
123 crate::reject_control_ids_in_ordinary_segment(
124 &token_ids[start..],
125 &self.special_token_ids,
126 )?;
127 }
128 }
129 Ok(Encoding::Sp(token_ids))
130 }
131}
132
133const MAX_ENCODE_CHARS: usize = 400_000;
135
136const MAX_RUN_CHARS: usize = 25_000;
139
140fn is_python_space(c: char) -> bool {
143 c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
144}
145
146fn for_each_reference_piece<'a>(text: &'a str, mut f: impl FnMut(&'a str)) {
153 if text.len() <= MAX_RUN_CHARS {
155 f(text);
156 return;
157 }
158 let mut chunk_start = 0;
159 let mut chunk_chars = 0;
160 for (index, _) in text.char_indices() {
161 if chunk_chars == MAX_ENCODE_CHARS {
162 split_long_runs(&text[chunk_start..index], &mut f);
163 chunk_start = index;
164 chunk_chars = 0;
165 }
166 chunk_chars += 1;
167 }
168 split_long_runs(&text[chunk_start..], &mut f);
169}
170
171fn split_long_runs<'a>(chunk: &'a str, f: &mut impl FnMut(&'a str)) {
174 let mut run_is_space = chunk.chars().next().is_some_and(is_python_space);
175 let mut run_len = 0;
176 let mut piece_start = 0;
177 for (index, c) in chunk.char_indices() {
178 let is_space = is_python_space(c);
179 if is_space != run_is_space {
180 run_is_space = is_space;
181 run_len = 1;
182 } else {
183 run_len += 1;
184 if run_len > MAX_RUN_CHARS {
185 f(&chunk[piece_start..index]);
186 piece_start = index;
187 run_len = 1;
188 }
189 }
190 }
191 f(&chunk[piece_start..]);
192}
193
194impl Decoder for TikTokenTokenizer {
195 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
196 let ids: Vec<u32> = if skip_special_tokens {
197 token_ids
198 .iter()
199 .filter(|&&id| !self.special_token_ids.contains(&id))
200 .copied()
201 .collect()
202 } else {
203 token_ids.to_vec()
204 };
205
206 let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
215 match String::from_utf8(bytes) {
216 Ok(text) => Ok(DecodeResult::Complete(text)),
217 Err(e) => {
218 let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
219 Ok(DecodeResult::from_decoded(text))
220 }
221 }
222 }
223}
224
225impl Tokenizer for TikTokenTokenizer {
226 fn validate_prefix_cache(&self) -> Result<()> {
227 Ok(())
228 }
229
230 fn vocab_size(&self) -> Option<usize> {
235 None
236 }
237
238 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
239 let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
240 ids.sort_unstable();
241 Ok(ids)
242 }
243
244 fn num_special_tokens_added(&self) -> Result<usize> {
248 Ok(0)
249 }
250}
251
252pub(crate) fn parse_tiktoken_file(path: &str) -> Result<FxHashMap<Vec<u8>, u32>> {
254 let contents = std::fs::read_to_string(path)
255 .map_err(|err| Error::msg(format!("Failed to read tiktoken file '{path}': {err}")))?;
256
257 let engine = base64::engine::general_purpose::STANDARD;
258 let mut encoder = FxHashMap::default();
259
260 for line in contents.lines() {
261 let line = line.trim();
262 if line.is_empty() {
263 continue;
264 }
265 let mut parts = line.split_whitespace();
266 let token_b64 = parts
267 .next()
268 .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no token): {line}")))?;
269 let rank_str = parts
270 .next()
271 .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no rank): {line}")))?;
272
273 let token_bytes = engine
274 .decode(token_b64)
275 .map_err(|err| Error::msg(format!("Invalid base64 in tiktoken file: {err}")))?;
276 let rank: u32 = rank_str
277 .parse()
278 .map_err(|err| Error::msg(format!("Invalid rank in tiktoken file: {err}")))?;
279
280 encoder.insert(token_bytes, rank);
281 }
282
283 Ok(encoder)
284}
285
286pub(crate) fn detect_bpe_pattern(directory: &Path) -> Result<&'static str> {
288 let model_type: String = crate::file_json_field(&directory.join("config.json"), "model_type")
289 .map_err(|err| {
290 Error::msg(format!("Failed to read model_type from config.json: {err}"))
291 })?;
292
293 match model_type.as_str() {
294 "kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "kimi_k31" | "kimi_linear"
300 | "deepseek_v3" => Ok(KIMI_PATTERN),
301 _ => Err(Error::msg(format!(
302 "Unsupported tiktoken model_type '{model_type}'. \
303 Currently supported: kimi, kimi_k2, kimi_k25, kimi_k3, kimi_k31, kimi_linear, deepseek_v3. \
304 To add a new model type, extend detect_bpe_pattern() in lib/tokenizers/src/tiktoken.rs \
305 with the appropriate BPE regex pattern. \
306 Alternatively, provide a tokenizer.json (HuggingFace format) instead."
307 ))),
308 }
309}
310
311pub(crate) struct AddedTokens {
313 pub(crate) tokens: FxHashMap<String, u32>,
315 pub(crate) skip_ids: HashSet<u32>,
318}
319
320pub(crate) fn load_added_tokens(directory: &Path, num_base_tokens: usize) -> Result<AddedTokens> {
325 let config_path = directory.join("tokenizer_config.json");
326 let mut tokens = FxHashMap::default();
327 let mut skip_ids = HashSet::new();
328
329 if !config_path.exists() {
330 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
332 let id = num_base_tokens as u32 + i;
333 tokens.insert(format!("<|reserved_token_{id}|>"), id);
334 skip_ids.insert(id);
335 }
336 return Ok(AddedTokens { tokens, skip_ids });
337 }
338
339 let contents = std::fs::read_to_string(&config_path)
340 .map_err(|err| Error::msg(format!("Failed to read tokenizer_config.json: {err}")))?;
341
342 let config: serde_json::Value = serde_json::from_str(&contents)
343 .map_err(|err| Error::msg(format!("Failed to parse tokenizer_config.json: {err}")))?;
344
345 if let Some(added_tokens) = config
346 .get("added_tokens_decoder")
347 .and_then(|v| v.as_object())
348 {
349 for (id_str, token_def) in added_tokens {
350 let id: u32 = id_str.parse().map_err(|err| {
351 Error::msg(format!(
352 "Invalid token ID '{id_str}' in added_tokens_decoder: {err}"
353 ))
354 })?;
355
356 let content = token_def
357 .get("content")
358 .and_then(|v| v.as_str())
359 .unwrap_or_else(|| {
360 tracing::warn!("Missing 'content' field for token ID {id}");
362 ""
363 });
364
365 if !content.is_empty() {
366 tokens.insert(content.to_string(), id);
367 if token_def
368 .get("special")
369 .and_then(serde_json::Value::as_bool)
370 == Some(true)
371 {
372 skip_ids.insert(id);
373 }
374 }
375 }
376
377 let used_ids: HashSet<u32> = tokens.values().copied().collect();
379 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
380 let id = num_base_tokens as u32 + i;
381 if !used_ids.contains(&id) {
382 tokens.insert(format!("<|reserved_token_{id}|>"), id);
383 skip_ids.insert(id);
384 }
385 }
386 } else {
387 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
389 let id = num_base_tokens as u32 + i;
390 tokens.insert(format!("<|reserved_token_{id}|>"), id);
391 skip_ids.insert(id);
392 }
393 }
394
395 Ok(AddedTokens { tokens, skip_ids })
396}
397
398#[cfg(test)]
399mod tests {
400 use super::*;
401 use crate::DecodeStream;
402 use std::io::Write;
403 use std::sync::Arc;
404
405 fn create_test_tiktoken_file(dir: &Path) -> String {
406 let engine = base64::engine::general_purpose::STANDARD;
407 let mut content = String::new();
408
409 let tokens: Vec<(&[u8], u32)> = vec![
411 (b"h", 0),
412 (b"e", 1),
413 (b"l", 2),
414 (b"o", 3),
415 (b" ", 4),
416 (b"w", 5),
417 (b"r", 6),
418 (b"d", 7),
419 (b"he", 8),
420 (b"ll", 9),
421 (b"lo", 10),
422 (b"wo", 11),
423 (b"rl", 12),
424 (b"hel", 13),
425 (b"llo", 14),
426 (b"wor", 15),
427 (b"hell", 16),
428 (b"ello", 17),
429 (b"worl", 18),
430 (b"hello", 19),
431 (b"world", 20),
432 ];
433
434 for (token, rank) in tokens {
435 let encoded = engine.encode(token);
436 content.push_str(&format!("{encoded} {rank}\n"));
437 }
438
439 let file_path = dir.join("tiktoken.model");
440 let mut file = std::fs::File::create(&file_path).unwrap();
441 file.write_all(content.as_bytes()).unwrap();
442 file_path.to_str().unwrap().to_string()
443 }
444
445 fn create_test_config(dir: &Path, model_type: &str) {
446 let config = serde_json::json!({
447 "model_type": model_type,
448 "max_position_embeddings": 32768,
449 "eos_token_id": [21]
450 });
451 let file_path = dir.join("config.json");
452 std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
453 }
454
455 fn create_test_tokenizer_config(dir: &Path, num_base_tokens: usize) {
456 let mut added_tokens = serde_json::Map::new();
457 let bos_id = num_base_tokens;
458 let eos_id = num_base_tokens + 1;
459
460 added_tokens.insert(
461 bos_id.to_string(),
462 serde_json::json!({"content": "[BOS]", "special": true}),
463 );
464 added_tokens.insert(
465 eos_id.to_string(),
466 serde_json::json!({"content": "[EOS]", "special": true}),
467 );
468
469 let config = serde_json::json!({
470 "added_tokens_decoder": added_tokens
471 });
472
473 let file_path = dir.join("tokenizer_config.json");
474 std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
475 }
476
477 #[test]
478 fn test_parse_tiktoken_file() {
479 let dir = tempfile::tempdir().unwrap();
480 let file_path = create_test_tiktoken_file(dir.path());
481 let encoder = parse_tiktoken_file(&file_path).unwrap();
482 assert_eq!(encoder.len(), 21);
483 assert_eq!(encoder[b"hello".as_slice()], 19);
484 assert_eq!(encoder[b"world".as_slice()], 20);
485 }
486
487 #[test]
488 fn test_parse_tiktoken_file_missing() {
489 let result = parse_tiktoken_file("/nonexistent/path/tiktoken.model");
490 assert!(result.is_err());
491 }
492
493 #[test]
494 fn test_tiktoken_from_file() {
495 let dir = tempfile::tempdir().unwrap();
496 let file_path = create_test_tiktoken_file(dir.path());
497
498 let mut special_tokens = FxHashMap::default();
499 special_tokens.insert("[BOS]".to_string(), 21_u32);
500 special_tokens.insert("[EOS]".to_string(), 22_u32);
501
502 let pattern = r"[\w]+|[^\w\s]+|\s+";
504
505 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
506
507 assert_eq!(
508 tokenizer.special_tokens(),
509 &["[BOS]".to_string(), "[EOS]".to_string()]
510 );
511
512 let encoding = tokenizer.encode("hello world").unwrap();
514 let ids = encoding.token_ids();
515 assert!(!ids.is_empty());
516
517 let decoded: String = tokenizer.decode(ids, false).unwrap().into();
519 assert_eq!(decoded, "hello world");
520 }
521
522 #[test]
523 fn test_tiktoken_encoding_variant() {
524 let dir = tempfile::tempdir().unwrap();
525 let file_path = create_test_tiktoken_file(dir.path());
526
527 let special_tokens = FxHashMap::default();
528 let pattern = r"[\w]+|[^\w\s]+|\s+";
529
530 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
531 assert!(tokenizer.special_tokens().is_empty());
532 let encoding = tokenizer.encode("hello").unwrap();
533
534 match &encoding {
536 Encoding::Sp(_) => {}
537 other => panic!("Expected Encoding::Sp, got {:?}", other),
538 }
539 }
540
541 #[test]
542 fn test_tiktoken_skip_special_tokens() {
543 let dir = tempfile::tempdir().unwrap();
544 let file_path = create_test_tiktoken_file(dir.path());
545
546 let mut special_tokens = FxHashMap::default();
547 special_tokens.insert("[BOS]".to_string(), 21_u32);
548 special_tokens.insert("[EOS]".to_string(), 22_u32);
549
550 let pattern = r"[\w]+|[^\w\s]+|\s+";
551
552 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
553
554 let encoding = tokenizer.encode("hello").unwrap();
556 let mut ids = vec![21u32]; ids.extend(encoding.token_ids());
558 ids.push(22); let decoded_skip: String = tokenizer.decode(&ids, true).unwrap().into();
562 assert_eq!(decoded_skip, "hello");
563
564 let decoded_all: String = tokenizer.decode(&ids, false).unwrap().into();
566 assert!(decoded_all.contains("hello"));
567 }
568
569 #[test]
570 fn test_special_token_ids_reports_configured_ids_sorted() {
571 let dir = tempfile::tempdir().unwrap();
572 let file_path = create_test_tiktoken_file(dir.path());
573
574 let mut special_tokens = FxHashMap::default();
575 special_tokens.insert("[EOS]".to_string(), 22_u32);
576 special_tokens.insert("[BOS]".to_string(), 21_u32);
577
578 let pattern = r"[\w]+|[^\w\s]+|\s+";
579 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
580
581 assert_eq!(tokenizer.special_token_ids().unwrap(), vec![21, 22]);
582 }
583
584 #[test]
585 fn test_vocab_size_and_token_to_id_are_unsupported() {
586 let dir = tempfile::tempdir().unwrap();
587 let file_path = create_test_tiktoken_file(dir.path());
588 let pattern = r"[\w]+|[^\w\s]+|\s+";
589 let tokenizer =
590 TikTokenTokenizer::from_file(&file_path, pattern, FxHashMap::default()).unwrap();
591
592 assert_eq!(tokenizer.vocab_size(), None);
593 assert!(tokenizer.token_to_id("hello").is_err());
594 }
595
596 #[test]
597 fn test_tiktoken_from_file_auto() {
598 let dir = tempfile::tempdir().unwrap();
599 let file_path = create_test_tiktoken_file(dir.path());
600
601 create_test_config(dir.path(), "kimi");
602 create_test_tokenizer_config(dir.path(), 21);
603
604 let mut expected_specials: Vec<String> = load_added_tokens(dir.path(), 21)
605 .unwrap()
606 .tokens
607 .into_keys()
608 .collect();
609 expected_specials.sort();
610 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
611 assert_eq!(tokenizer.special_tokens(), expected_specials);
612
613 let encoding = tokenizer.encode("hello world").unwrap();
615 let ids = encoding.token_ids();
616 assert!(!ids.is_empty());
617
618 let decoded: String = tokenizer.decode(ids, false).unwrap().into();
619 assert_eq!(decoded, "hello world");
620
621 create_test_config(dir.path(), "kimi_k31");
623 let k31_tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
624 assert_eq!(
625 k31_tokenizer.encode("hello world").unwrap().token_ids(),
626 ids
627 );
628 let decoded: String = k31_tokenizer.decode(ids, false).unwrap().into();
629 assert_eq!(decoded, "hello world");
630 }
631
632 #[test]
633 fn test_detect_bpe_pattern_unknown() {
634 let dir = tempfile::tempdir().unwrap();
635 create_test_config(dir.path(), "unknown_model");
636 let result = detect_bpe_pattern(dir.path());
637 assert!(result.is_err());
638 }
639
640 #[test]
641 fn test_kimi_k3_uses_kimi_bpe_pattern() {
642 let dir = tempfile::tempdir().unwrap();
643 for model_type in ["kimi_k3", "kimi_k31"] {
644 create_test_config(dir.path(), model_type);
645 assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
646 }
647 }
648
649 #[test]
650 fn test_detect_bpe_pattern_kimi_linear() {
651 let dir = tempfile::tempdir().unwrap();
652 create_test_config(dir.path(), "kimi_linear");
653 assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
654 }
655
656 #[test]
657 fn test_load_special_tokens_no_config() {
658 let dir = tempfile::tempdir().unwrap();
659 let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
660 assert_eq!(tokens.len(), 256);
661 assert_eq!(tokens["<|reserved_token_100|>"], 100);
662 assert_eq!(tokens["<|reserved_token_355|>"], 355);
663 }
664
665 #[test]
666 fn test_load_special_tokens_with_config() {
667 let dir = tempfile::tempdir().unwrap();
668 create_test_tokenizer_config(dir.path(), 100);
669 let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
670 assert_eq!(tokens["[BOS]"], 100);
671 assert_eq!(tokens["[EOS]"], 101);
672 assert!(tokens.len() > 2);
674 }
675
676 fn create_tokenizer_with_non_special_added_tokens(dir: &Path) -> TikTokenTokenizer {
679 let file_path = create_test_tiktoken_file(dir);
680 create_test_config(dir, "kimi_k3");
681 let config = serde_json::json!({"added_tokens_decoder": {
682 "21": {"content": "[BOS]", "special": true},
683 "22": {"content": "<|open|>", "special": false},
684 "23": {"content": "<|sep|>"},
685 }});
686 std::fs::write(dir.join("tokenizer_config.json"), config.to_string()).unwrap();
687 TikTokenTokenizer::from_file_auto(&file_path).unwrap()
688 }
689
690 #[test]
691 fn test_skip_special_tokens_keeps_non_special_added_tokens() {
692 let dir = tempfile::tempdir().unwrap();
693 let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
694
695 let encoding = tokenizer.encode("<|open|>hello").unwrap();
696 assert_eq!(encoding.token_ids(), &[22, 19]);
697
698 let special_ids = tokenizer.special_token_ids().unwrap();
699 assert!(special_ids.contains(&21));
700 assert!(special_ids.contains(&24), "reserved placeholder");
701 assert!(!special_ids.contains(&22));
702 assert!(!special_ids.contains(&23));
703
704 let ids = [21, 22, 19, 23];
705 let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
706 assert_eq!(skipped, "<|open|>hello<|sep|>");
707 let kept: String = tokenizer.decode(&ids, false).unwrap().into();
708 assert_eq!(kept, "[BOS]<|open|>hello<|sep|>");
709 }
710
711 #[test]
712 fn test_decode_stream_emits_leading_non_special_added_token() {
713 let dir = tempfile::tempdir().unwrap();
714 let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
715 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
716
717 let mut stream = DecodeStream::new(tokenizer_arc, &[19], true);
718 assert_eq!(stream.step(22).unwrap().as_deref(), Some("<|open|>"));
719 assert_eq!(stream.step(19).unwrap().as_deref(), Some("hello"));
720 assert_eq!(stream.step(21).unwrap(), None);
721 assert_eq!(stream.step(23).unwrap().as_deref(), Some("<|sep|>"));
722 }
723
724 fn create_test_tiktoken_file_with_byte_tokens(dir: &Path) -> String {
726 let engine = base64::engine::general_purpose::STANDARD;
727 let mut content = String::new();
728
729 let tokens: Vec<(&[u8], u32)> = vec![
730 (b"h", 0),
731 (b"e", 1),
732 (b"l", 2),
733 (b"o", 3),
734 (b" ", 4),
735 (b"hello", 5),
736 ];
737
738 for (token, rank) in &tokens {
739 let encoded = engine.encode(token);
740 content.push_str(&format!("{encoded} {rank}\n"));
741 }
742
743 let byte_tokens: Vec<(Vec<u8>, u32)> =
746 vec![(vec![0xE4], 100), (vec![0xBD], 101), (vec![0xA0], 102)];
747
748 for (token, rank) in &byte_tokens {
749 let encoded = engine.encode(token);
750 content.push_str(&format!("{encoded} {rank}\n"));
751 }
752
753 let emoji_tokens: Vec<(Vec<u8>, u32)> = vec![
755 (vec![0xF0], 200),
756 (vec![0x9F], 201),
757 (vec![0x98], 202),
758 (vec![0x80], 203),
759 ];
760
761 for (token, rank) in &emoji_tokens {
762 let encoded = engine.encode(token);
763 content.push_str(&format!("{encoded} {rank}\n"));
764 }
765
766 let fffd_token: Vec<(Vec<u8>, u32)> = vec![(vec![0xEF, 0xBF, 0xBD], 300)];
769
770 for (token, rank) in &fffd_token {
771 let encoded = engine.encode(token);
772 content.push_str(&format!("{encoded} {rank}\n"));
773 }
774
775 let file_path = dir.join("tiktoken.model");
776 let mut file = std::fs::File::create(&file_path).unwrap();
777 file.write_all(content.as_bytes()).unwrap();
778 file_path.to_str().unwrap().to_string()
779 }
780
781 fn create_byte_token_tokenizer(dir: &Path) -> TikTokenTokenizer {
782 let file_path = create_test_tiktoken_file_with_byte_tokens(dir);
783 let special_tokens = FxHashMap::default();
784 let pattern = r"[\w]+|[^\w\s]+|\s+";
785 TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap()
786 }
787
788 #[test]
792 fn test_decode_single_incomplete_utf8_byte_does_not_error() {
793 let dir = tempfile::tempdir().unwrap();
794 let tokenizer = create_byte_token_tokenizer(dir.path());
795
796 let result = tokenizer.decode(&[100], false);
797 assert!(
798 result.is_ok(),
799 "decode() should not error on incomplete UTF-8 bytes"
800 );
801 let decode_result = result.unwrap();
802 assert!(
803 decode_result.is_partial(),
804 "incomplete UTF-8 byte should produce DecodeResult::Partial, got: {:?}",
805 decode_result
806 );
807 }
808
809 #[test]
811 fn test_decode_two_of_three_utf8_bytes_does_not_error() {
812 let dir = tempfile::tempdir().unwrap();
813 let tokenizer = create_byte_token_tokenizer(dir.path());
814
815 let result = tokenizer.decode(&[100, 101], false);
816 assert!(result.is_ok());
817 let decode_result = result.unwrap();
818 assert!(
819 decode_result.is_partial(),
820 "incomplete 2-of-3 UTF-8 bytes should produce DecodeResult::Partial, got: {:?}",
821 decode_result
822 );
823 }
824
825 #[test]
829 fn test_decode_complete_multibyte_utf8_produces_correct_char() {
830 let dir = tempfile::tempdir().unwrap();
831 let tokenizer = create_byte_token_tokenizer(dir.path());
832
833 let result = tokenizer.decode(&[100, 101, 102], false);
834 assert!(result.is_ok());
835 assert_eq!(String::from(result.unwrap()), "δ½ ");
836 }
837
838 #[test]
841 fn test_decode_complete_4byte_emoji_from_byte_tokens() {
842 let dir = tempfile::tempdir().unwrap();
843 let tokenizer = create_byte_token_tokenizer(dir.path());
844
845 let result = tokenizer.decode(&[200, 201, 202, 203], false);
846 assert!(result.is_ok());
847 assert_eq!(String::from(result.unwrap()), "π");
848 }
849
850 #[test]
855 fn test_decode_legitimate_replacement_char_token_is_complete() {
856 let dir = tempfile::tempdir().unwrap();
857 let tokenizer = create_byte_token_tokenizer(dir.path());
858
859 let result = tokenizer.decode(&[300], false);
860 assert!(result.is_ok());
861 let decode_result = result.unwrap();
862 assert!(
863 decode_result.is_complete(),
864 "legitimate U+FFFD vocab token must be Complete, got: {:?}",
865 decode_result
866 );
867 assert_eq!(decode_result.as_str(), "\u{FFFD}");
868 }
869
870 #[test]
872 fn test_decode_partial_emoji_does_not_error() {
873 let dir = tempfile::tempdir().unwrap();
874 let tokenizer = create_byte_token_tokenizer(dir.path());
875
876 let result = tokenizer.decode(&[200], false);
877 assert!(result.is_ok());
878 assert!(result.unwrap().is_partial());
879 }
880
881 #[test]
883 fn test_decode_mixed_ascii_and_incomplete_bytes() {
884 let dir = tempfile::tempdir().unwrap();
885 let tokenizer = create_byte_token_tokenizer(dir.path());
886
887 let result = tokenizer.decode(&[5, 100], false);
888 assert!(result.is_ok());
889 let decode_result = result.unwrap();
890 assert!(
891 decode_result.is_partial(),
892 "trailing incomplete byte should produce DecodeResult::Partial"
893 );
894 let text: String = decode_result.into();
895 assert!(
896 text.starts_with("hello"),
897 "should start with 'hello', got: {:?}",
898 text
899 );
900 }
901
902 #[test]
906 fn test_decode_stream_incremental_multibyte_reassembly() {
907 let dir = tempfile::tempdir().unwrap();
908 let tokenizer = create_byte_token_tokenizer(dir.path());
909 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
910
911 let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
912
913 let r1 = stream.step(100).unwrap();
914 assert_eq!(r1, None, "first byte of 3-byte char should be buffered");
915
916 let r2 = stream.step(101).unwrap();
917 assert_eq!(r2, None, "second byte of 3-byte char should be buffered");
918
919 let r3 = stream.step(102).unwrap();
920 assert!(r3.is_some(), "third byte should complete the character");
921 assert_eq!(r3.unwrap(), "δ½ ");
922 }
923
924 #[test]
926 fn test_decode_stream_incremental_emoji_reassembly() {
927 let dir = tempfile::tempdir().unwrap();
928 let tokenizer = create_byte_token_tokenizer(dir.path());
929 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
930
931 let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
932
933 let r1 = stream.step(200).unwrap();
934 assert_eq!(r1, None, "byte 1/4 of emoji should be buffered");
935
936 let r2 = stream.step(201).unwrap();
937 assert_eq!(r2, None, "byte 2/4 of emoji should be buffered");
938
939 let r3 = stream.step(202).unwrap();
940 assert_eq!(r3, None, "byte 3/4 of emoji should be buffered");
941
942 let r4 = stream.step(203).unwrap();
943 assert!(r4.is_some(), "byte 4/4 should complete the emoji");
944 assert_eq!(r4.unwrap(), "π");
945 }
946
947 #[test]
948 fn test_tiktoken_encode_batch() {
949 let dir = tempfile::tempdir().unwrap();
950 let file_path = create_test_tiktoken_file(dir.path());
951
952 let special_tokens = FxHashMap::default();
953 let pattern = r"[\w]+|[^\w\s]+|\s+";
954
955 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
956
957 let inputs = &["hello", "world"];
958 let encodings = tokenizer.encode_batch(inputs).unwrap();
959 assert_eq!(encodings.len(), 2);
960
961 for (encoding, input) in encodings.iter().zip(inputs.iter()) {
962 let decoded: String = tokenizer
963 .decode(encoding.token_ids(), false)
964 .unwrap()
965 .into();
966 assert_eq!(decoded, *input);
967 }
968 }
969
970 fn create_byte_level_tiktoken_file(dir: &Path) -> String {
973 let engine = base64::engine::general_purpose::STANDARD;
974 let mut content = String::new();
975 for byte_val in 0u16..256 {
976 let encoded = engine.encode([byte_val as u8]);
977 content.push_str(&format!("{encoded} {byte_val}\n"));
978 }
979 let file_path = dir.join("tiktoken.model");
980 std::fs::write(&file_path, &content).unwrap();
981 file_path.to_str().unwrap().to_string()
982 }
983
984 #[test]
985 fn test_segment_encoding_preserves_special_vs_ordinary_boundary() {
986 let dir = tempfile::tempdir().unwrap();
987 let file_path = create_byte_level_tiktoken_file(dir.path());
988 let marker = "<|open|>";
989 let marker_id = 256;
990 let mut special_tokens = FxHashMap::default();
991 special_tokens.insert(marker.to_string(), marker_id);
992 let tokenizer =
993 TikTokenTokenizer::from_file(&file_path, r"(?s).+", special_tokens).unwrap();
994
995 let structural = tokenizer
996 .encode_segments(&[EncodeSegment::control(marker)])
997 .unwrap();
998 let user_text = tokenizer
999 .encode_segments(&[EncodeSegment::ordinary(marker)])
1000 .unwrap();
1001
1002 assert_eq!(structural.token_ids(), &[marker_id]);
1003 assert_ne!(user_text.token_ids(), &[marker_id]);
1004 let decoded: String = tokenizer
1005 .decode(user_text.token_ids(), false)
1006 .unwrap()
1007 .into();
1008 assert_eq!(decoded, marker);
1009 }
1010
1011 fn piece_chars(text: &str) -> Vec<usize> {
1014 let mut lengths = Vec::new();
1015 for_each_reference_piece(text, |piece| lengths.push(piece.chars().count()));
1016 lengths
1017 }
1018
1019 #[test]
1020 fn test_reference_pieces_match_kimi_reference_split() {
1021 assert_eq!(piece_chars("hello world"), vec![11]);
1024 assert_eq!(
1025 piece_chars(&"a".repeat(60_000)),
1026 vec![25_000, 25_000, 10_000]
1027 );
1028 assert_eq!(
1029 piece_chars(&("a".repeat(30_000) + &" ".repeat(30_000))),
1030 vec![25_000, 30_000, 5_000]
1031 );
1032 assert_eq!(
1033 piece_chars(&("x".repeat(25_000) + " " + &"y".repeat(25_001))),
1034 vec![50_001, 1]
1035 );
1036 assert_eq!(
1040 piece_chars(&("a".repeat(20_000) + &"\u{1c}".repeat(10_000))),
1041 vec![30_000]
1042 );
1043 assert_eq!(piece_chars(&"Γ©".repeat(26_000)), vec![25_000, 1_000]);
1045 let chunked: String = "ab ".repeat(136_667).chars().take(410_000).collect();
1047 assert_eq!(piece_chars(&chunked), vec![400_000, 10_000]);
1048 }
1049
1050 #[test]
1051 fn test_segment_encoding_cuts_long_runs_like_the_reference() {
1052 let dir = tempfile::tempdir().unwrap();
1054 let engine = base64::engine::general_purpose::STANDARD;
1055 let mut content = String::new();
1056 for byte_val in 0u16..256 {
1057 let encoded = engine.encode([byte_val as u8]);
1058 content.push_str(&format!("{encoded} {byte_val}\n"));
1059 }
1060 content.push_str(&format!("{} 256\n", engine.encode(b"aa")));
1061 let file_path = dir.path().join("tiktoken.model");
1062 std::fs::write(&file_path, &content).unwrap();
1063 let tokenizer = TikTokenTokenizer::from_file(
1064 file_path.to_str().unwrap(),
1065 r"(?s).+",
1066 FxHashMap::default(),
1067 )
1068 .unwrap();
1069
1070 let text = "b".to_string() + &"a".repeat(25_000);
1073 let segmented = tokenizer
1074 .encode_segments(&[EncodeSegment::ordinary(&text)])
1075 .unwrap();
1076 let mut expected = tokenizer.bpe.encode_ordinary(&text[..25_000]);
1077 expected.extend(tokenizer.bpe.encode_ordinary(&text[25_000..]));
1078 assert_eq!(segmented.token_ids(), expected.as_slice());
1079 assert_ne!(
1080 segmented.token_ids(),
1081 tokenizer.bpe.encode_ordinary(&text).as_slice(),
1082 "the cut must change ids for this input, or the test proves nothing"
1083 );
1084 }
1085
1086 fn has_nontrivial_self_overlap(token: &str) -> bool {
1087 let bytes = token.as_bytes();
1088 (1..bytes.len()).any(|overlap| bytes[bytes.len() - overlap..] == bytes[..overlap])
1089 }
1090
1091 fn have_ambiguous_overlap(a: &str, b: &str) -> bool {
1092 let a = a.as_bytes();
1093 let b = b.as_bytes();
1094
1095 if a.windows(b.len()).any(|window| window == b)
1096 || b.windows(a.len()).any(|window| window == a)
1097 {
1098 return true;
1099 }
1100
1101 let max_overlap = a.len().min(b.len());
1102 (1..max_overlap).any(|overlap| {
1103 a[a.len() - overlap..] == b[..overlap] || b[b.len() - overlap..] == a[..overlap]
1104 })
1105 }
1106
1107 fn assert_special_split_invariant(
1108 tokenizer: &TikTokenTokenizer,
1109 left: &str,
1110 special: &str,
1111 right: &str,
1112 ) {
1113 let whole = tokenizer
1114 .encode(&format!("{left}{special}{right}"))
1115 .unwrap()
1116 .token_ids()
1117 .to_vec();
1118 let mut split = tokenizer
1119 .encode(&format!("{left}{special}"))
1120 .unwrap()
1121 .token_ids()
1122 .to_vec();
1123 split.extend_from_slice(tokenizer.encode(right).unwrap().token_ids());
1124 assert_eq!(
1125 whole, split,
1126 "splitting after registered special token {special:?} must preserve tokenization"
1127 );
1128 }
1129
1130 #[test]
1131 fn test_registered_special_tokens_are_cache_safe_boundaries() {
1132 let dir = tempfile::tempdir().unwrap();
1133 let file_path = create_byte_level_tiktoken_file(dir.path());
1134 create_test_config(dir.path(), "kimi");
1135 create_test_tokenizer_config(dir.path(), 256);
1136
1137 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1138 let specials = tokenizer.special_tokens();
1139 assert!(!specials.is_empty());
1140
1141 for (index, special) in specials.iter().enumerate() {
1142 assert!(
1143 !has_nontrivial_self_overlap(special),
1144 "registered special token has an ambiguous self-overlap: {special:?}"
1145 );
1146 for other in &specials[index + 1..] {
1147 assert!(
1148 !have_ambiguous_overlap(special, other),
1149 "registered special tokens overlap ambiguously: {special:?} and {other:?}"
1150 );
1151 }
1152
1153 for (left, right) in [
1154 ("ordinary prefix ", " ordinary suffix"),
1155 ("line before\n", "\tUnicode after: εδΊ¬ π"),
1156 ("", " trailing text"),
1157 ] {
1158 assert_special_split_invariant(&tokenizer, left, special, right);
1159 }
1160 }
1161
1162 let bos = specials
1163 .iter()
1164 .find(|token| token.as_str() == "[BOS]")
1165 .unwrap();
1166 let eos = specials
1167 .iter()
1168 .find(|token| token.as_str() == "[EOS]")
1169 .unwrap();
1170 assert_special_split_invariant(
1171 &tokenizer,
1172 "prefix ",
1173 bos,
1174 &format!("{eos}<|reserved_token_258|>Unicode ε°Ύι¨"),
1175 );
1176 }
1177
1178 #[test]
1189 fn test_reserved_token_absolute_id_naming_kimi_k25_regression() {
1190 let dir = tempfile::tempdir().unwrap();
1191 let file_path = create_byte_level_tiktoken_file(dir.path());
1192
1193 create_test_config(dir.path(), "kimi");
1195
1196 create_test_tokenizer_config(dir.path(), 256);
1198
1199 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1200
1201 let single = "<|reserved_token_258|>";
1208 let enc = tokenizer.encode(single).unwrap();
1209 assert_eq!(
1210 enc.token_ids().len(),
1211 1,
1212 "'{single}' should be 1 special token, got {} tokens: {:?}. \
1213 This means fallback naming still uses relative offsets instead of absolute IDs.",
1214 enc.token_ids().len(),
1215 enc.token_ids()
1216 );
1217 assert_eq!(enc.token_ids()[0], 258);
1218
1219 let multi: String = (258u32..268)
1222 .map(|id| format!("<|reserved_token_{id}|>"))
1223 .collect();
1224 let enc_multi = tokenizer.encode(&multi).unwrap();
1225 assert_eq!(
1226 enc_multi.token_ids().len(),
1227 10,
1228 "10 reserved token strings should produce exactly 10 tokens, got {}: {:?}",
1229 enc_multi.token_ids().len(),
1230 enc_multi.token_ids()
1231 );
1232 let expected_ids: Vec<u32> = (258..268).collect();
1233 assert_eq!(enc_multi.token_ids(), &expected_ids);
1234 }
1235
1236 #[test]
1240 fn test_relative_offset_naming_causes_inflation() {
1241 let dir = tempfile::tempdir().unwrap();
1242 let file_path = create_byte_level_tiktoken_file(dir.path());
1243
1244 let _encoder = parse_tiktoken_file(&file_path).unwrap();
1245 let num_base_tokens = 256usize;
1246
1247 let mut bad_special_tokens: FxHashMap<String, u32> = FxHashMap::default();
1249 bad_special_tokens.insert("[BOS]".to_string(), 256);
1250 bad_special_tokens.insert("[EOS]".to_string(), 257);
1251 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
1252 let id = num_base_tokens as u32 + i;
1253 if id != 256 && id != 257 {
1254 bad_special_tokens.insert(format!("<|reserved_token_{i}|>"), id);
1256 }
1257 }
1258
1259 let bad_tokenizer =
1260 TikTokenTokenizer::from_file(&file_path, KIMI_PATTERN, bad_special_tokens).unwrap();
1261
1262 let input = "<|reserved_token_258|>";
1265 let enc = bad_tokenizer.encode(input).unwrap();
1266 assert!(
1267 enc.token_ids().len() > 1,
1268 "With buggy relative-offset naming, '{}' should NOT be recognized as a \
1269 single special token. Got {} token(s): {:?}",
1270 input,
1271 enc.token_ids().len(),
1272 enc.token_ids()
1273 );
1274
1275 let multi: String = (258u32..268)
1277 .map(|id| format!("<|reserved_token_{id}|>"))
1278 .collect();
1279 let enc_multi = bad_tokenizer.encode(&multi).unwrap();
1280 assert!(
1281 enc_multi.token_ids().len() > 10,
1282 "With buggy naming, 10 reserved token strings should inflate beyond 10 tokens. \
1283 Got {}",
1284 enc_multi.token_ids().len(),
1285 );
1286 }
1287}