1use std::collections::HashSet;
5use std::path::Path;
6
7use base64::Engine as _;
8use rayon::prelude::*;
9use rustc_hash::FxHashMap;
10use tiktoken_rs::CoreBPE;
11
12use super::{
13 EncodeSegment, Encoding, Error, Result, TokenIdType,
14 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17const DEFAULT_NUM_RESERVED_SPECIAL_TOKENS: u32 = 256;
20
21const KIMI_PATTERN: &str = fastokens::tiktoken::KIMI_PATTERN;
23
24pub struct TikTokenTokenizer {
25 bpe: CoreBPE,
26 special_token_ids: HashSet<u32>,
27 special_tokens: Vec<String>,
28}
29
30pub(crate) fn sorted_special_token_strings(special_tokens: &FxHashMap<String, u32>) -> Vec<String> {
31 let mut strings: Vec<String> = special_tokens.keys().cloned().collect();
32 strings.sort();
33 strings.dedup();
34 strings
35}
36
37impl TikTokenTokenizer {
38 pub fn from_file(
45 path: &str,
46 pattern: &str,
47 special_tokens: FxHashMap<String, u32>,
48 ) -> Result<Self> {
49 let encoder = parse_tiktoken_file(path)?;
50 let special_token_ids: HashSet<u32> = special_tokens.values().copied().collect();
51 let special_token_strings = sorted_special_token_strings(&special_tokens);
52
53 let bpe = CoreBPE::new(encoder, special_tokens, pattern)
54 .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
55
56 Ok(Self {
57 bpe,
58 special_token_ids,
59 special_tokens: special_token_strings,
60 })
61 }
62
63 pub fn from_file_auto(path: &str) -> Result<Self> {
68 let file_path = Path::new(path);
69 let directory = file_path
70 .parent()
71 .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
72
73 let pattern = detect_bpe_pattern(directory)?;
74 let encoder = parse_tiktoken_file(path)?;
75 let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
77 let AddedTokens { tokens, skip_ids } = load_added_tokens(directory, num_base_tokens)?;
78 let special_token_strings = sorted_special_token_strings(&tokens);
79
80 let bpe = CoreBPE::new(encoder, tokens, pattern)
81 .map_err(|err| Error::msg(format!("Error creating tiktoken BPE: {err}")))?;
82
83 Ok(Self {
84 bpe,
85 special_token_ids: skip_ids,
86 special_tokens: special_token_strings,
87 })
88 }
89
90 pub fn special_tokens(&self) -> &[String] {
97 &self.special_tokens
98 }
99}
100
101impl Encoder for TikTokenTokenizer {
102 fn encode(&self, input: &str) -> Result<Encoding> {
103 let token_ids: Vec<u32> = self.bpe.encode_with_special_tokens(input);
104 Ok(Encoding::Sp(token_ids))
105 }
106
107 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
108 inputs.par_iter().map(|input| self.encode(input)).collect()
109 }
110
111 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
112 let mut token_ids = Vec::new();
113 for segment in segments {
114 if segment.allow_special {
115 token_ids.extend(self.bpe.encode_with_special_tokens(segment.text));
116 } else {
117 token_ids.extend(self.bpe.encode_ordinary(segment.text));
118 }
119 }
120 Ok(Encoding::Sp(token_ids))
121 }
122}
123
124impl Decoder for TikTokenTokenizer {
125 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
126 let ids: Vec<u32> = if skip_special_tokens {
127 token_ids
128 .iter()
129 .filter(|&&id| !self.special_token_ids.contains(&id))
130 .copied()
131 .collect()
132 } else {
133 token_ids.to_vec()
134 };
135
136 let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
145 match String::from_utf8(bytes) {
146 Ok(text) => Ok(DecodeResult::Complete(text)),
147 Err(e) => {
148 let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
149 Ok(DecodeResult::from_decoded(text))
150 }
151 }
152 }
153}
154
155impl Tokenizer for TikTokenTokenizer {
156 fn validate_prefix_cache(&self) -> Result<()> {
157 Ok(())
158 }
159
160 fn vocab_size(&self) -> Option<usize> {
165 None
166 }
167
168 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
169 let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
170 ids.sort_unstable();
171 Ok(ids)
172 }
173
174 fn num_special_tokens_added(&self) -> Result<usize> {
178 Ok(0)
179 }
180}
181
182pub(crate) fn parse_tiktoken_file(path: &str) -> Result<FxHashMap<Vec<u8>, u32>> {
184 let contents = std::fs::read_to_string(path)
185 .map_err(|err| Error::msg(format!("Failed to read tiktoken file '{path}': {err}")))?;
186
187 let engine = base64::engine::general_purpose::STANDARD;
188 let mut encoder = FxHashMap::default();
189
190 for line in contents.lines() {
191 let line = line.trim();
192 if line.is_empty() {
193 continue;
194 }
195 let mut parts = line.split_whitespace();
196 let token_b64 = parts
197 .next()
198 .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no token): {line}")))?;
199 let rank_str = parts
200 .next()
201 .ok_or_else(|| Error::msg(format!("Invalid tiktoken line (no rank): {line}")))?;
202
203 let token_bytes = engine
204 .decode(token_b64)
205 .map_err(|err| Error::msg(format!("Invalid base64 in tiktoken file: {err}")))?;
206 let rank: u32 = rank_str
207 .parse()
208 .map_err(|err| Error::msg(format!("Invalid rank in tiktoken file: {err}")))?;
209
210 encoder.insert(token_bytes, rank);
211 }
212
213 Ok(encoder)
214}
215
216pub(crate) fn detect_bpe_pattern(directory: &Path) -> Result<&'static str> {
218 let model_type: String = crate::file_json_field(&directory.join("config.json"), "model_type")
219 .map_err(|err| {
220 Error::msg(format!("Failed to read model_type from config.json: {err}"))
221 })?;
222
223 match model_type.as_str() {
224 "kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "kimi_linear" | "deepseek_v3" => {
230 Ok(KIMI_PATTERN)
231 }
232 _ => Err(Error::msg(format!(
233 "Unsupported tiktoken model_type '{model_type}'. \
234 Currently supported: kimi, kimi_k2, kimi_k25, kimi_k3, kimi_linear, deepseek_v3. \
235 To add a new model type, extend detect_bpe_pattern() in lib/tokenizers/src/tiktoken.rs \
236 with the appropriate BPE regex pattern. \
237 Alternatively, provide a tokenizer.json (HuggingFace format) instead."
238 ))),
239 }
240}
241
242pub(crate) struct AddedTokens {
244 pub(crate) tokens: FxHashMap<String, u32>,
246 pub(crate) skip_ids: HashSet<u32>,
249}
250
251pub(crate) fn load_added_tokens(directory: &Path, num_base_tokens: usize) -> Result<AddedTokens> {
256 let config_path = directory.join("tokenizer_config.json");
257 let mut tokens = FxHashMap::default();
258 let mut skip_ids = HashSet::new();
259
260 if !config_path.exists() {
261 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
263 let id = num_base_tokens as u32 + i;
264 tokens.insert(format!("<|reserved_token_{id}|>"), id);
265 skip_ids.insert(id);
266 }
267 return Ok(AddedTokens { tokens, skip_ids });
268 }
269
270 let contents = std::fs::read_to_string(&config_path)
271 .map_err(|err| Error::msg(format!("Failed to read tokenizer_config.json: {err}")))?;
272
273 let config: serde_json::Value = serde_json::from_str(&contents)
274 .map_err(|err| Error::msg(format!("Failed to parse tokenizer_config.json: {err}")))?;
275
276 if let Some(added_tokens) = config
277 .get("added_tokens_decoder")
278 .and_then(|v| v.as_object())
279 {
280 for (id_str, token_def) in added_tokens {
281 let id: u32 = id_str.parse().map_err(|err| {
282 Error::msg(format!(
283 "Invalid token ID '{id_str}' in added_tokens_decoder: {err}"
284 ))
285 })?;
286
287 let content = token_def
288 .get("content")
289 .and_then(|v| v.as_str())
290 .unwrap_or_else(|| {
291 tracing::warn!("Missing 'content' field for token ID {id}");
293 ""
294 });
295
296 if !content.is_empty() {
297 tokens.insert(content.to_string(), id);
298 if token_def
299 .get("special")
300 .and_then(serde_json::Value::as_bool)
301 == Some(true)
302 {
303 skip_ids.insert(id);
304 }
305 }
306 }
307
308 let used_ids: HashSet<u32> = tokens.values().copied().collect();
310 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
311 let id = num_base_tokens as u32 + i;
312 if !used_ids.contains(&id) {
313 tokens.insert(format!("<|reserved_token_{id}|>"), id);
314 skip_ids.insert(id);
315 }
316 }
317 } else {
318 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
320 let id = num_base_tokens as u32 + i;
321 tokens.insert(format!("<|reserved_token_{id}|>"), id);
322 skip_ids.insert(id);
323 }
324 }
325
326 Ok(AddedTokens { tokens, skip_ids })
327}
328
329#[cfg(test)]
330mod tests {
331 use super::*;
332 use crate::DecodeStream;
333 use std::io::Write;
334 use std::sync::Arc;
335
336 fn create_test_tiktoken_file(dir: &Path) -> String {
337 let engine = base64::engine::general_purpose::STANDARD;
338 let mut content = String::new();
339
340 let tokens: Vec<(&[u8], u32)> = vec![
342 (b"h", 0),
343 (b"e", 1),
344 (b"l", 2),
345 (b"o", 3),
346 (b" ", 4),
347 (b"w", 5),
348 (b"r", 6),
349 (b"d", 7),
350 (b"he", 8),
351 (b"ll", 9),
352 (b"lo", 10),
353 (b"wo", 11),
354 (b"rl", 12),
355 (b"hel", 13),
356 (b"llo", 14),
357 (b"wor", 15),
358 (b"hell", 16),
359 (b"ello", 17),
360 (b"worl", 18),
361 (b"hello", 19),
362 (b"world", 20),
363 ];
364
365 for (token, rank) in tokens {
366 let encoded = engine.encode(token);
367 content.push_str(&format!("{encoded} {rank}\n"));
368 }
369
370 let file_path = dir.join("tiktoken.model");
371 let mut file = std::fs::File::create(&file_path).unwrap();
372 file.write_all(content.as_bytes()).unwrap();
373 file_path.to_str().unwrap().to_string()
374 }
375
376 fn create_test_config(dir: &Path, model_type: &str) {
377 let config = serde_json::json!({
378 "model_type": model_type,
379 "max_position_embeddings": 32768,
380 "eos_token_id": [21]
381 });
382 let file_path = dir.join("config.json");
383 std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
384 }
385
386 fn create_test_tokenizer_config(dir: &Path, num_base_tokens: usize) {
387 let mut added_tokens = serde_json::Map::new();
388 let bos_id = num_base_tokens;
389 let eos_id = num_base_tokens + 1;
390
391 added_tokens.insert(
392 bos_id.to_string(),
393 serde_json::json!({"content": "[BOS]", "special": true}),
394 );
395 added_tokens.insert(
396 eos_id.to_string(),
397 serde_json::json!({"content": "[EOS]", "special": true}),
398 );
399
400 let config = serde_json::json!({
401 "added_tokens_decoder": added_tokens
402 });
403
404 let file_path = dir.join("tokenizer_config.json");
405 std::fs::write(file_path, serde_json::to_string_pretty(&config).unwrap()).unwrap();
406 }
407
408 #[test]
409 fn test_parse_tiktoken_file() {
410 let dir = tempfile::tempdir().unwrap();
411 let file_path = create_test_tiktoken_file(dir.path());
412 let encoder = parse_tiktoken_file(&file_path).unwrap();
413 assert_eq!(encoder.len(), 21);
414 assert_eq!(encoder[b"hello".as_slice()], 19);
415 assert_eq!(encoder[b"world".as_slice()], 20);
416 }
417
418 #[test]
419 fn test_parse_tiktoken_file_missing() {
420 let result = parse_tiktoken_file("/nonexistent/path/tiktoken.model");
421 assert!(result.is_err());
422 }
423
424 #[test]
425 fn test_tiktoken_from_file() {
426 let dir = tempfile::tempdir().unwrap();
427 let file_path = create_test_tiktoken_file(dir.path());
428
429 let mut special_tokens = FxHashMap::default();
430 special_tokens.insert("[BOS]".to_string(), 21_u32);
431 special_tokens.insert("[EOS]".to_string(), 22_u32);
432
433 let pattern = r"[\w]+|[^\w\s]+|\s+";
435
436 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
437
438 assert_eq!(
439 tokenizer.special_tokens(),
440 &["[BOS]".to_string(), "[EOS]".to_string()]
441 );
442
443 let encoding = tokenizer.encode("hello world").unwrap();
445 let ids = encoding.token_ids();
446 assert!(!ids.is_empty());
447
448 let decoded: String = tokenizer.decode(ids, false).unwrap().into();
450 assert_eq!(decoded, "hello world");
451 }
452
453 #[test]
454 fn test_tiktoken_encoding_variant() {
455 let dir = tempfile::tempdir().unwrap();
456 let file_path = create_test_tiktoken_file(dir.path());
457
458 let special_tokens = FxHashMap::default();
459 let pattern = r"[\w]+|[^\w\s]+|\s+";
460
461 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
462 assert!(tokenizer.special_tokens().is_empty());
463 let encoding = tokenizer.encode("hello").unwrap();
464
465 match &encoding {
467 Encoding::Sp(_) => {}
468 other => panic!("Expected Encoding::Sp, got {:?}", other),
469 }
470 }
471
472 #[test]
473 fn test_tiktoken_skip_special_tokens() {
474 let dir = tempfile::tempdir().unwrap();
475 let file_path = create_test_tiktoken_file(dir.path());
476
477 let mut special_tokens = FxHashMap::default();
478 special_tokens.insert("[BOS]".to_string(), 21_u32);
479 special_tokens.insert("[EOS]".to_string(), 22_u32);
480
481 let pattern = r"[\w]+|[^\w\s]+|\s+";
482
483 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
484
485 let encoding = tokenizer.encode("hello").unwrap();
487 let mut ids = vec![21u32]; ids.extend(encoding.token_ids());
489 ids.push(22); let decoded_skip: String = tokenizer.decode(&ids, true).unwrap().into();
493 assert_eq!(decoded_skip, "hello");
494
495 let decoded_all: String = tokenizer.decode(&ids, false).unwrap().into();
497 assert!(decoded_all.contains("hello"));
498 }
499
500 #[test]
501 fn test_special_token_ids_reports_configured_ids_sorted() {
502 let dir = tempfile::tempdir().unwrap();
503 let file_path = create_test_tiktoken_file(dir.path());
504
505 let mut special_tokens = FxHashMap::default();
506 special_tokens.insert("[EOS]".to_string(), 22_u32);
507 special_tokens.insert("[BOS]".to_string(), 21_u32);
508
509 let pattern = r"[\w]+|[^\w\s]+|\s+";
510 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
511
512 assert_eq!(tokenizer.special_token_ids().unwrap(), vec![21, 22]);
513 }
514
515 #[test]
516 fn test_vocab_size_and_token_to_id_are_unsupported() {
517 let dir = tempfile::tempdir().unwrap();
518 let file_path = create_test_tiktoken_file(dir.path());
519 let pattern = r"[\w]+|[^\w\s]+|\s+";
520 let tokenizer =
521 TikTokenTokenizer::from_file(&file_path, pattern, FxHashMap::default()).unwrap();
522
523 assert_eq!(tokenizer.vocab_size(), None);
524 assert!(tokenizer.token_to_id("hello").is_err());
525 }
526
527 #[test]
528 fn test_tiktoken_from_file_auto() {
529 let dir = tempfile::tempdir().unwrap();
530 let file_path = create_test_tiktoken_file(dir.path());
531
532 create_test_config(dir.path(), "kimi");
533 create_test_tokenizer_config(dir.path(), 21);
534
535 let mut expected_specials: Vec<String> = load_added_tokens(dir.path(), 21)
536 .unwrap()
537 .tokens
538 .into_keys()
539 .collect();
540 expected_specials.sort();
541 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
542 assert_eq!(tokenizer.special_tokens(), expected_specials);
543
544 let encoding = tokenizer.encode("hello world").unwrap();
546 let ids = encoding.token_ids();
547 assert!(!ids.is_empty());
548
549 let decoded: String = tokenizer.decode(ids, false).unwrap().into();
550 assert_eq!(decoded, "hello world");
551 }
552
553 #[test]
554 fn test_detect_bpe_pattern_unknown() {
555 let dir = tempfile::tempdir().unwrap();
556 create_test_config(dir.path(), "unknown_model");
557 let result = detect_bpe_pattern(dir.path());
558 assert!(result.is_err());
559 }
560
561 #[test]
562 fn test_kimi_k3_uses_kimi_bpe_pattern() {
563 let dir = tempfile::tempdir().unwrap();
564 create_test_config(dir.path(), "kimi_k3");
565 assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
566 }
567
568 #[test]
569 fn test_detect_bpe_pattern_kimi_linear() {
570 let dir = tempfile::tempdir().unwrap();
571 create_test_config(dir.path(), "kimi_linear");
572 assert_eq!(detect_bpe_pattern(dir.path()).unwrap(), KIMI_PATTERN);
573 }
574
575 #[test]
576 fn test_load_special_tokens_no_config() {
577 let dir = tempfile::tempdir().unwrap();
578 let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
579 assert_eq!(tokens.len(), 256);
580 assert_eq!(tokens["<|reserved_token_100|>"], 100);
581 assert_eq!(tokens["<|reserved_token_355|>"], 355);
582 }
583
584 #[test]
585 fn test_load_special_tokens_with_config() {
586 let dir = tempfile::tempdir().unwrap();
587 create_test_tokenizer_config(dir.path(), 100);
588 let tokens = load_added_tokens(dir.path(), 100).unwrap().tokens;
589 assert_eq!(tokens["[BOS]"], 100);
590 assert_eq!(tokens["[EOS]"], 101);
591 assert!(tokens.len() > 2);
593 }
594
595 fn create_tokenizer_with_non_special_added_tokens(dir: &Path) -> TikTokenTokenizer {
598 let file_path = create_test_tiktoken_file(dir);
599 create_test_config(dir, "kimi_k3");
600 let config = serde_json::json!({"added_tokens_decoder": {
601 "21": {"content": "[BOS]", "special": true},
602 "22": {"content": "<|open|>", "special": false},
603 "23": {"content": "<|sep|>"},
604 }});
605 std::fs::write(dir.join("tokenizer_config.json"), config.to_string()).unwrap();
606 TikTokenTokenizer::from_file_auto(&file_path).unwrap()
607 }
608
609 #[test]
610 fn test_skip_special_tokens_keeps_non_special_added_tokens() {
611 let dir = tempfile::tempdir().unwrap();
612 let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
613
614 let encoding = tokenizer.encode("<|open|>hello").unwrap();
615 assert_eq!(encoding.token_ids(), &[22, 19]);
616
617 let special_ids = tokenizer.special_token_ids().unwrap();
618 assert!(special_ids.contains(&21));
619 assert!(special_ids.contains(&24), "reserved placeholder");
620 assert!(!special_ids.contains(&22));
621 assert!(!special_ids.contains(&23));
622
623 let ids = [21, 22, 19, 23];
624 let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
625 assert_eq!(skipped, "<|open|>hello<|sep|>");
626 let kept: String = tokenizer.decode(&ids, false).unwrap().into();
627 assert_eq!(kept, "[BOS]<|open|>hello<|sep|>");
628 }
629
630 #[test]
631 fn test_decode_stream_emits_leading_non_special_added_token() {
632 let dir = tempfile::tempdir().unwrap();
633 let tokenizer = create_tokenizer_with_non_special_added_tokens(dir.path());
634 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
635
636 let mut stream = DecodeStream::new(tokenizer_arc, &[19], true);
637 assert_eq!(stream.step(22).unwrap().as_deref(), Some("<|open|>"));
638 assert_eq!(stream.step(19).unwrap().as_deref(), Some("hello"));
639 assert_eq!(stream.step(21).unwrap(), None);
640 assert_eq!(stream.step(23).unwrap().as_deref(), Some("<|sep|>"));
641 }
642
643 fn create_test_tiktoken_file_with_byte_tokens(dir: &Path) -> String {
645 let engine = base64::engine::general_purpose::STANDARD;
646 let mut content = String::new();
647
648 let tokens: Vec<(&[u8], u32)> = vec![
649 (b"h", 0),
650 (b"e", 1),
651 (b"l", 2),
652 (b"o", 3),
653 (b" ", 4),
654 (b"hello", 5),
655 ];
656
657 for (token, rank) in &tokens {
658 let encoded = engine.encode(token);
659 content.push_str(&format!("{encoded} {rank}\n"));
660 }
661
662 let byte_tokens: Vec<(Vec<u8>, u32)> =
665 vec![(vec![0xE4], 100), (vec![0xBD], 101), (vec![0xA0], 102)];
666
667 for (token, rank) in &byte_tokens {
668 let encoded = engine.encode(token);
669 content.push_str(&format!("{encoded} {rank}\n"));
670 }
671
672 let emoji_tokens: Vec<(Vec<u8>, u32)> = vec![
674 (vec![0xF0], 200),
675 (vec![0x9F], 201),
676 (vec![0x98], 202),
677 (vec![0x80], 203),
678 ];
679
680 for (token, rank) in &emoji_tokens {
681 let encoded = engine.encode(token);
682 content.push_str(&format!("{encoded} {rank}\n"));
683 }
684
685 let fffd_token: Vec<(Vec<u8>, u32)> = vec![(vec![0xEF, 0xBF, 0xBD], 300)];
688
689 for (token, rank) in &fffd_token {
690 let encoded = engine.encode(token);
691 content.push_str(&format!("{encoded} {rank}\n"));
692 }
693
694 let file_path = dir.join("tiktoken.model");
695 let mut file = std::fs::File::create(&file_path).unwrap();
696 file.write_all(content.as_bytes()).unwrap();
697 file_path.to_str().unwrap().to_string()
698 }
699
700 fn create_byte_token_tokenizer(dir: &Path) -> TikTokenTokenizer {
701 let file_path = create_test_tiktoken_file_with_byte_tokens(dir);
702 let special_tokens = FxHashMap::default();
703 let pattern = r"[\w]+|[^\w\s]+|\s+";
704 TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap()
705 }
706
707 #[test]
711 fn test_decode_single_incomplete_utf8_byte_does_not_error() {
712 let dir = tempfile::tempdir().unwrap();
713 let tokenizer = create_byte_token_tokenizer(dir.path());
714
715 let result = tokenizer.decode(&[100], false);
716 assert!(
717 result.is_ok(),
718 "decode() should not error on incomplete UTF-8 bytes"
719 );
720 let decode_result = result.unwrap();
721 assert!(
722 decode_result.is_partial(),
723 "incomplete UTF-8 byte should produce DecodeResult::Partial, got: {:?}",
724 decode_result
725 );
726 }
727
728 #[test]
730 fn test_decode_two_of_three_utf8_bytes_does_not_error() {
731 let dir = tempfile::tempdir().unwrap();
732 let tokenizer = create_byte_token_tokenizer(dir.path());
733
734 let result = tokenizer.decode(&[100, 101], false);
735 assert!(result.is_ok());
736 let decode_result = result.unwrap();
737 assert!(
738 decode_result.is_partial(),
739 "incomplete 2-of-3 UTF-8 bytes should produce DecodeResult::Partial, got: {:?}",
740 decode_result
741 );
742 }
743
744 #[test]
748 fn test_decode_complete_multibyte_utf8_produces_correct_char() {
749 let dir = tempfile::tempdir().unwrap();
750 let tokenizer = create_byte_token_tokenizer(dir.path());
751
752 let result = tokenizer.decode(&[100, 101, 102], false);
753 assert!(result.is_ok());
754 assert_eq!(String::from(result.unwrap()), "δ½ ");
755 }
756
757 #[test]
760 fn test_decode_complete_4byte_emoji_from_byte_tokens() {
761 let dir = tempfile::tempdir().unwrap();
762 let tokenizer = create_byte_token_tokenizer(dir.path());
763
764 let result = tokenizer.decode(&[200, 201, 202, 203], false);
765 assert!(result.is_ok());
766 assert_eq!(String::from(result.unwrap()), "π");
767 }
768
769 #[test]
774 fn test_decode_legitimate_replacement_char_token_is_complete() {
775 let dir = tempfile::tempdir().unwrap();
776 let tokenizer = create_byte_token_tokenizer(dir.path());
777
778 let result = tokenizer.decode(&[300], false);
779 assert!(result.is_ok());
780 let decode_result = result.unwrap();
781 assert!(
782 decode_result.is_complete(),
783 "legitimate U+FFFD vocab token must be Complete, got: {:?}",
784 decode_result
785 );
786 assert_eq!(decode_result.as_str(), "\u{FFFD}");
787 }
788
789 #[test]
791 fn test_decode_partial_emoji_does_not_error() {
792 let dir = tempfile::tempdir().unwrap();
793 let tokenizer = create_byte_token_tokenizer(dir.path());
794
795 let result = tokenizer.decode(&[200], false);
796 assert!(result.is_ok());
797 assert!(result.unwrap().is_partial());
798 }
799
800 #[test]
802 fn test_decode_mixed_ascii_and_incomplete_bytes() {
803 let dir = tempfile::tempdir().unwrap();
804 let tokenizer = create_byte_token_tokenizer(dir.path());
805
806 let result = tokenizer.decode(&[5, 100], false);
807 assert!(result.is_ok());
808 let decode_result = result.unwrap();
809 assert!(
810 decode_result.is_partial(),
811 "trailing incomplete byte should produce DecodeResult::Partial"
812 );
813 let text: String = decode_result.into();
814 assert!(
815 text.starts_with("hello"),
816 "should start with 'hello', got: {:?}",
817 text
818 );
819 }
820
821 #[test]
825 fn test_decode_stream_incremental_multibyte_reassembly() {
826 let dir = tempfile::tempdir().unwrap();
827 let tokenizer = create_byte_token_tokenizer(dir.path());
828 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
829
830 let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
831
832 let r1 = stream.step(100).unwrap();
833 assert_eq!(r1, None, "first byte of 3-byte char should be buffered");
834
835 let r2 = stream.step(101).unwrap();
836 assert_eq!(r2, None, "second byte of 3-byte char should be buffered");
837
838 let r3 = stream.step(102).unwrap();
839 assert!(r3.is_some(), "third byte should complete the character");
840 assert_eq!(r3.unwrap(), "δ½ ");
841 }
842
843 #[test]
845 fn test_decode_stream_incremental_emoji_reassembly() {
846 let dir = tempfile::tempdir().unwrap();
847 let tokenizer = create_byte_token_tokenizer(dir.path());
848 let tokenizer_arc: Arc<dyn crate::traits::Tokenizer> = Arc::new(tokenizer);
849
850 let mut stream = DecodeStream::new(tokenizer_arc, &[5], false);
851
852 let r1 = stream.step(200).unwrap();
853 assert_eq!(r1, None, "byte 1/4 of emoji should be buffered");
854
855 let r2 = stream.step(201).unwrap();
856 assert_eq!(r2, None, "byte 2/4 of emoji should be buffered");
857
858 let r3 = stream.step(202).unwrap();
859 assert_eq!(r3, None, "byte 3/4 of emoji should be buffered");
860
861 let r4 = stream.step(203).unwrap();
862 assert!(r4.is_some(), "byte 4/4 should complete the emoji");
863 assert_eq!(r4.unwrap(), "π");
864 }
865
866 #[test]
867 fn test_tiktoken_encode_batch() {
868 let dir = tempfile::tempdir().unwrap();
869 let file_path = create_test_tiktoken_file(dir.path());
870
871 let special_tokens = FxHashMap::default();
872 let pattern = r"[\w]+|[^\w\s]+|\s+";
873
874 let tokenizer = TikTokenTokenizer::from_file(&file_path, pattern, special_tokens).unwrap();
875
876 let inputs = &["hello", "world"];
877 let encodings = tokenizer.encode_batch(inputs).unwrap();
878 assert_eq!(encodings.len(), 2);
879
880 for (encoding, input) in encodings.iter().zip(inputs.iter()) {
881 let decoded: String = tokenizer
882 .decode(encoding.token_ids(), false)
883 .unwrap()
884 .into();
885 assert_eq!(decoded, *input);
886 }
887 }
888
889 fn create_byte_level_tiktoken_file(dir: &Path) -> String {
892 let engine = base64::engine::general_purpose::STANDARD;
893 let mut content = String::new();
894 for byte_val in 0u16..256 {
895 let encoded = engine.encode([byte_val as u8]);
896 content.push_str(&format!("{encoded} {byte_val}\n"));
897 }
898 let file_path = dir.join("tiktoken.model");
899 std::fs::write(&file_path, &content).unwrap();
900 file_path.to_str().unwrap().to_string()
901 }
902
903 #[test]
904 fn test_segment_encoding_preserves_special_vs_ordinary_boundary() {
905 let dir = tempfile::tempdir().unwrap();
906 let file_path = create_byte_level_tiktoken_file(dir.path());
907 let marker = "<|open|>";
908 let marker_id = 256;
909 let mut special_tokens = FxHashMap::default();
910 special_tokens.insert(marker.to_string(), marker_id);
911 let tokenizer =
912 TikTokenTokenizer::from_file(&file_path, r"(?s).+", special_tokens).unwrap();
913
914 let structural = tokenizer
915 .encode_segments(&[EncodeSegment::control(marker)])
916 .unwrap();
917 let user_text = tokenizer
918 .encode_segments(&[EncodeSegment::ordinary(marker)])
919 .unwrap();
920
921 assert_eq!(structural.token_ids(), &[marker_id]);
922 assert_ne!(user_text.token_ids(), &[marker_id]);
923 let decoded: String = tokenizer
924 .decode(user_text.token_ids(), false)
925 .unwrap()
926 .into();
927 assert_eq!(decoded, marker);
928 }
929
930 fn has_nontrivial_self_overlap(token: &str) -> bool {
931 let bytes = token.as_bytes();
932 (1..bytes.len()).any(|overlap| bytes[bytes.len() - overlap..] == bytes[..overlap])
933 }
934
935 fn have_ambiguous_overlap(a: &str, b: &str) -> bool {
936 let a = a.as_bytes();
937 let b = b.as_bytes();
938
939 if a.windows(b.len()).any(|window| window == b)
940 || b.windows(a.len()).any(|window| window == a)
941 {
942 return true;
943 }
944
945 let max_overlap = a.len().min(b.len());
946 (1..max_overlap).any(|overlap| {
947 a[a.len() - overlap..] == b[..overlap] || b[b.len() - overlap..] == a[..overlap]
948 })
949 }
950
951 fn assert_special_split_invariant(
952 tokenizer: &TikTokenTokenizer,
953 left: &str,
954 special: &str,
955 right: &str,
956 ) {
957 let whole = tokenizer
958 .encode(&format!("{left}{special}{right}"))
959 .unwrap()
960 .token_ids()
961 .to_vec();
962 let mut split = tokenizer
963 .encode(&format!("{left}{special}"))
964 .unwrap()
965 .token_ids()
966 .to_vec();
967 split.extend_from_slice(tokenizer.encode(right).unwrap().token_ids());
968 assert_eq!(
969 whole, split,
970 "splitting after registered special token {special:?} must preserve tokenization"
971 );
972 }
973
974 #[test]
975 fn test_registered_special_tokens_are_cache_safe_boundaries() {
976 let dir = tempfile::tempdir().unwrap();
977 let file_path = create_byte_level_tiktoken_file(dir.path());
978 create_test_config(dir.path(), "kimi");
979 create_test_tokenizer_config(dir.path(), 256);
980
981 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
982 let specials = tokenizer.special_tokens();
983 assert!(!specials.is_empty());
984
985 for (index, special) in specials.iter().enumerate() {
986 assert!(
987 !has_nontrivial_self_overlap(special),
988 "registered special token has an ambiguous self-overlap: {special:?}"
989 );
990 for other in &specials[index + 1..] {
991 assert!(
992 !have_ambiguous_overlap(special, other),
993 "registered special tokens overlap ambiguously: {special:?} and {other:?}"
994 );
995 }
996
997 for (left, right) in [
998 ("ordinary prefix ", " ordinary suffix"),
999 ("line before\n", "\tUnicode after: εδΊ¬ π"),
1000 ("", " trailing text"),
1001 ] {
1002 assert_special_split_invariant(&tokenizer, left, special, right);
1003 }
1004 }
1005
1006 let bos = specials
1007 .iter()
1008 .find(|token| token.as_str() == "[BOS]")
1009 .unwrap();
1010 let eos = specials
1011 .iter()
1012 .find(|token| token.as_str() == "[EOS]")
1013 .unwrap();
1014 assert_special_split_invariant(
1015 &tokenizer,
1016 "prefix ",
1017 bos,
1018 &format!("{eos}<|reserved_token_258|>Unicode ε°Ύι¨"),
1019 );
1020 }
1021
1022 #[test]
1033 fn test_reserved_token_absolute_id_naming_kimi_k25_regression() {
1034 let dir = tempfile::tempdir().unwrap();
1035 let file_path = create_byte_level_tiktoken_file(dir.path());
1036
1037 create_test_config(dir.path(), "kimi");
1039
1040 create_test_tokenizer_config(dir.path(), 256);
1042
1043 let tokenizer = TikTokenTokenizer::from_file_auto(&file_path).unwrap();
1044
1045 let single = "<|reserved_token_258|>";
1052 let enc = tokenizer.encode(single).unwrap();
1053 assert_eq!(
1054 enc.token_ids().len(),
1055 1,
1056 "'{single}' should be 1 special token, got {} tokens: {:?}. \
1057 This means fallback naming still uses relative offsets instead of absolute IDs.",
1058 enc.token_ids().len(),
1059 enc.token_ids()
1060 );
1061 assert_eq!(enc.token_ids()[0], 258);
1062
1063 let multi: String = (258u32..268)
1066 .map(|id| format!("<|reserved_token_{id}|>"))
1067 .collect();
1068 let enc_multi = tokenizer.encode(&multi).unwrap();
1069 assert_eq!(
1070 enc_multi.token_ids().len(),
1071 10,
1072 "10 reserved token strings should produce exactly 10 tokens, got {}: {:?}",
1073 enc_multi.token_ids().len(),
1074 enc_multi.token_ids()
1075 );
1076 let expected_ids: Vec<u32> = (258..268).collect();
1077 assert_eq!(enc_multi.token_ids(), &expected_ids);
1078 }
1079
1080 #[test]
1084 fn test_relative_offset_naming_causes_inflation() {
1085 let dir = tempfile::tempdir().unwrap();
1086 let file_path = create_byte_level_tiktoken_file(dir.path());
1087
1088 let _encoder = parse_tiktoken_file(&file_path).unwrap();
1089 let num_base_tokens = 256usize;
1090
1091 let mut bad_special_tokens: FxHashMap<String, u32> = FxHashMap::default();
1093 bad_special_tokens.insert("[BOS]".to_string(), 256);
1094 bad_special_tokens.insert("[EOS]".to_string(), 257);
1095 for i in 0..DEFAULT_NUM_RESERVED_SPECIAL_TOKENS {
1096 let id = num_base_tokens as u32 + i;
1097 if id != 256 && id != 257 {
1098 bad_special_tokens.insert(format!("<|reserved_token_{i}|>"), id);
1100 }
1101 }
1102
1103 let bad_tokenizer =
1104 TikTokenTokenizer::from_file(&file_path, KIMI_PATTERN, bad_special_tokens).unwrap();
1105
1106 let input = "<|reserved_token_258|>";
1109 let enc = bad_tokenizer.encode(input).unwrap();
1110 assert!(
1111 enc.token_ids().len() > 1,
1112 "With buggy relative-offset naming, '{}' should NOT be recognized as a \
1113 single special token. Got {} token(s): {:?}",
1114 input,
1115 enc.token_ids().len(),
1116 enc.token_ids()
1117 );
1118
1119 let multi: String = (258u32..268)
1121 .map(|id| format!("<|reserved_token_{id}|>"))
1122 .collect();
1123 let enc_multi = bad_tokenizer.encode(&multi).unwrap();
1124 assert!(
1125 enc_multi.token_ids().len() > 10,
1126 "With buggy naming, 10 reserved token strings should inflate beyond 10 tokens. \
1127 Got {}",
1128 enc_multi.token_ids().len(),
1129 );
1130 }
1131}