1use std::collections::HashSet;
14use std::path::Path;
15
16use rayon::prelude::*;
17use rustc_hash::FxHashMap;
18
19use super::{
20 EncodeSegment, Encoding, Error, Result, TokenIdType,
21 hf::HuggingFaceTokenizer,
22 tiktoken,
23 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
24};
25
26fn fast_encode(encoder: &fastokens::Tokenizer, input: &str) -> Result<Encoding> {
27 let ids = encoder
28 .encode(input)
29 .map_err(|e| Error::msg(format!("Fastokens encode error: {e}")))?;
30 Ok(Encoding::Sp(ids))
31}
32
33fn fast_encode_segments(
34 encoder: &fastokens::Tokenizer,
35 segments: &[EncodeSegment<'_>],
36 control_ids: &HashSet<u32>,
37) -> Result<Encoding> {
38 let mut ids = Vec::new();
42 for segment in segments {
43 let start = ids.len();
44 let segment_ids: Vec<u32> = encoder
45 .encode_segments_unpadded(&[fastokens::EncodeSegment {
46 text: segment.text,
47 allow_special: segment.allow_special,
48 }])
49 .map_err(|e| Error::msg(format!("Fastokens segmented encode error: {e}")))?;
50 ids.extend(segment_ids);
51 if !segment.allow_special {
52 crate::reject_control_ids_in_ordinary_segment(&ids[start..], control_ids)?;
53 }
54 }
55 Ok(Encoding::Sp(ids))
56}
57
58pub struct FastTokenizer {
62 fast_encoder: fastokens::Tokenizer,
63 hf_decoder: HuggingFaceTokenizer,
64}
65
66impl FastTokenizer {
67 pub fn from_file(path: &str) -> Result<Self> {
72 let mut fast_encoder = fastokens::Tokenizer::from_file(Path::new(path))
73 .map_err(|e| Error::msg(format!("Error loading fastokens tokenizer: {e}")))?;
74 fast_encoder.set_truncation(None);
75 fast_encoder.set_padding(None);
76 let hf_decoder = HuggingFaceTokenizer::from_file(path)?;
77 Ok(Self {
78 fast_encoder,
79 hf_decoder,
80 })
81 }
82}
83
84impl Encoder for FastTokenizer {
85 fn encode(&self, input: &str) -> Result<Encoding> {
86 fast_encode(&self.fast_encoder, input)
87 }
88
89 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
90 inputs.par_iter().map(|input| self.encode(input)).collect()
91 }
92
93 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
94 fast_encode_segments(&self.fast_encoder, segments, self.hf_decoder.added_ids())
95 }
96}
97
98impl Decoder for FastTokenizer {
99 fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
100 self.hf_decoder
101 .has_unstable_suffix(token_ids, skip_special_tokens)
102 }
103
104 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
105 self.hf_decoder.decode(token_ids, skip_special_tokens)
106 }
107}
108
109impl Tokenizer for FastTokenizer {
110 fn validate_prefix_cache(&self) -> Result<()> {
111 Ok(())
112 }
113
114 fn validate_segment_prefix_cache(&self) -> Result<()> {
117 Ok(())
118 }
119
120 fn vocab_size(&self) -> Option<usize> {
123 self.hf_decoder.vocab_size()
124 }
125
126 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
127 self.hf_decoder.token_to_id(token)
128 }
129
130 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
131 self.hf_decoder.special_token_ids()
132 }
133
134 fn num_special_tokens_added(&self) -> Result<usize> {
135 Ok(0)
136 }
137}
138
139pub struct FastTikTokenTokenizer {
149 inner: fastokens::Tokenizer,
150 id_to_bytes: FxHashMap<u32, Vec<u8>>,
153 special_token_ids: HashSet<u32>,
154 special_tokens: Vec<String>,
155 added_ids: HashSet<u32>,
158}
159
160impl FastTikTokenTokenizer {
161 pub fn from_file_auto(path: &str) -> Result<Self> {
164 let directory = Path::new(path)
165 .parent()
166 .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
167 let pattern = tiktoken::detect_bpe_pattern(directory)?;
168 let encoder = tiktoken::parse_tiktoken_file(path)?;
169 let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
170 let tiktoken::AddedTokens { tokens, skip_ids } =
171 tiktoken::load_added_tokens(directory, num_base_tokens)?;
172
173 let ranks: Vec<(Vec<u8>, u32)> = encoder.into_iter().collect();
174 let mut id_to_bytes: FxHashMap<u32, Vec<u8>> = ranks
175 .iter()
176 .map(|(bytes, rank)| (*rank, bytes.clone()))
177 .collect();
178 id_to_bytes.extend(
179 tokens
180 .iter()
181 .map(|(content, &id)| (id, content.as_bytes().to_vec())),
182 );
183 let special_token_strings = tiktoken::sorted_special_token_strings(&tokens);
184 let added_ids = tokens.values().copied().collect();
185
186 let config =
187 fastokens::tiktoken::TiktokenConfig::new(pattern, tokens.into_iter().collect());
188 let inner = fastokens::Tokenizer::from_tiktoken_ranks(&ranks, config).map_err(|e| {
189 Error::msg(format!(
190 "Error loading fastokens tiktoken tokenizer from {path}: {e}"
191 ))
192 })?;
193 Ok(Self {
194 inner,
195 id_to_bytes,
196 special_token_ids: skip_ids,
197 special_tokens: special_token_strings,
198 added_ids,
199 })
200 }
201
202 pub fn special_tokens(&self) -> &[String] {
206 &self.special_tokens
207 }
208}
209
210impl Encoder for FastTikTokenTokenizer {
211 fn encode(&self, input: &str) -> Result<Encoding> {
212 fast_encode(&self.inner, input)
213 }
214
215 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
216 inputs.par_iter().map(|input| self.encode(input)).collect()
217 }
218
219 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
220 fast_encode_segments(&self.inner, segments, &self.added_ids)
221 }
222}
223
224impl Decoder for FastTikTokenTokenizer {
225 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
226 let mut bytes = Vec::new();
227 for id in token_ids {
228 if skip_special_tokens && self.special_token_ids.contains(id) {
229 continue;
230 }
231 if let Some(token) = self.id_to_bytes.get(id) {
232 bytes.extend_from_slice(token);
233 }
234 }
235 match String::from_utf8(bytes) {
236 Ok(text) => Ok(DecodeResult::Complete(text)),
237 Err(e) => Ok(DecodeResult::from_decoded(
238 String::from_utf8_lossy(e.as_bytes()).into_owned(),
239 )),
240 }
241 }
242}
243
244impl Tokenizer for FastTikTokenTokenizer {
245 fn validate_prefix_cache(&self) -> Result<()> {
246 Err(Error::msg(
247 "fastokens over tiktoken.model plain-text prefix caching is disabled pending \
248 validation of the full cache matrix",
249 ))
250 }
251
252 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
253 Ok(self.inner.token_to_id(token))
254 }
255
256 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
257 let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
258 ids.sort_unstable();
259 Ok(ids)
260 }
261
262 fn num_special_tokens_added(&self) -> Result<usize> {
263 Ok(0)
264 }
265}
266
267#[cfg(test)]
268mod tests {
269 use super::*;
270 use crate::{HuggingFaceTokenizer, TokenizerOptions};
271
272 const TOKENIZER_PATH: &str = concat!(
275 env!("CARGO_MANIFEST_DIR"),
276 "/tests/data/minimal-bpe/tokenizer.json"
277 );
278 const SEGMENTED_TOKENIZER_PATH: &str = concat!(
279 env!("CARGO_MANIFEST_DIR"),
280 "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
281 );
282
283 #[test]
284 fn byte_fallback_stream_matches_full_decode() {
285 let tokenizer: crate::Tokenizer =
286 std::sync::Arc::new(FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap()).into();
287 for (pieces, skip) in [
288 (vec!["<0x61>", "<0xF5>"], false),
289 (vec!["<0x61>", "</s>", "<0xF5>"], false),
290 (vec!["<0x61>", "</s>", "<0xF5>"], true),
291 ] {
292 let ids: Vec<_> = pieces
293 .iter()
294 .map(|piece| tokenizer.token_to_id(piece).unwrap().unwrap())
295 .collect();
296 let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
297 let mut stream = tokenizer.decode_stream(&[], skip);
298 let mut actual = String::new();
299 for id in ids {
300 actual.push_str(&stream.step(id).unwrap().unwrap_or_default());
301 }
302 actual.push_str(&stream.finish().unwrap().unwrap_or_default());
303 assert_eq!(actual, expected, "{pieces:?}, skip={skip}");
304 }
305 }
306
307 #[test]
308 fn test_fast_encode_decode_roundtrip() {
309 let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
310 let text = "Hello, world!";
315 let encoding = tokenizer.encode(text).unwrap();
316 assert!(!encoding.token_ids().is_empty());
317 let decoded: String = tokenizer.decode(encoding.token_ids(), true).unwrap().into();
318 assert!(!decoded.is_empty());
319 let enc_chars: String = text.chars().filter(|c| !c.is_whitespace()).collect();
321 let dec_chars: String = decoded.chars().filter(|c| !c.is_whitespace()).collect();
322 assert_eq!(
323 enc_chars, dec_chars,
324 "non-space characters must be preserved"
325 );
326 }
327
328 #[test]
329 fn test_fast_matches_hf_encoding() {
330 let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
331 let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
332
333 for text in &["Hello, world!", "Hello", " world", "He llo"] {
334 let fast_ids = fast.encode(text).unwrap();
335 let hf_ids = hf.encode(text).unwrap();
336 assert_eq!(
337 fast_ids.token_ids(),
338 hf_ids.token_ids(),
339 "fastokens and HuggingFace must produce identical token IDs for '{text}'"
340 );
341 }
342 }
343
344 #[test]
345 fn test_fast_batch_encode() {
346 let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
347 let inputs = &["Hello", " world", "Hello, world!"];
348 let encodings = tokenizer.encode_batch(inputs).unwrap();
349 assert_eq!(encodings.len(), inputs.len());
350 for (enc, input) in encodings.iter().zip(inputs.iter()) {
351 assert!(
352 !enc.token_ids().is_empty(),
353 "encoding for '{input}' must be non-empty"
354 );
355 }
356 }
357
358 #[test]
359 fn test_fast_segmented_encoding_preserves_trust_boundaries() {
360 let tokenizer = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
361 let upstream =
362 fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
363 .unwrap();
364 let marker = "<s>";
365
366 let trusted = tokenizer
367 .encode_segments(&[EncodeSegment::control(marker)])
368 .unwrap();
369 assert_eq!(
370 trusted.token_ids(),
371 &[upstream.token_to_id(marker).unwrap()],
372 "trusted renderer output must recognize the control token"
373 );
374
375 let ordinary = tokenizer
376 .encode_segments(&[EncodeSegment::ordinary(marker)])
377 .unwrap();
378 assert_ne!(
379 ordinary.token_ids(),
380 trusted.token_ids(),
381 "untrusted content must encode the control-token spelling as ordinary text"
382 );
383
384 let segments = [
385 EncodeSegment::ordinary("hello "),
386 EncodeSegment::control(marker),
387 EncodeSegment::ordinary(marker),
388 ];
389 let upstream_segments = [
390 fastokens::EncodeSegment::ordinary("hello "),
391 fastokens::EncodeSegment::special(marker),
392 fastokens::EncodeSegment::ordinary(marker),
393 ];
394 let actual = tokenizer.encode_segments(&segments).unwrap();
395 let expected = upstream.encode_segments(&upstream_segments).unwrap();
396 assert_eq!(actual.token_ids(), expected);
397
398 assert!(
399 tokenizer
400 .encode_segments(&[])
401 .unwrap()
402 .token_ids()
403 .is_empty()
404 );
405 }
406
407 #[test]
408 fn test_fast_segmented_encoding_rejects_ordinary_control_ids() {
409 let temp = tempfile::tempdir().unwrap();
412 let path = temp.path().join("tokenizer.json");
413 let mut json: serde_json::Value =
414 serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
415 json["added_tokens"] = serde_json::json!([{
416 "id": 18,
417 "content": "He",
418 "single_word": false,
419 "lstrip": false,
420 "rstrip": false,
421 "normalized": false,
422 "special": true
423 }]);
424 std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
425
426 let tokenizer = FastTokenizer::from_file(path.to_str().unwrap()).unwrap();
427 let control = tokenizer
428 .encode_segments(&[EncodeSegment::control("He")])
429 .unwrap();
430 assert_eq!(control.token_ids(), &[18]);
431 let error = tokenizer
432 .encode_segments(&[EncodeSegment::ordinary("He")])
433 .unwrap_err();
434 assert!(
435 error.to_string().contains("control-token id 18"),
436 "unexpected error: {error}"
437 );
438 }
439
440 #[test]
441 fn tokenizer_json_truncation_and_padding_are_ignored() {
442 let temp = tempfile::tempdir().unwrap();
443 let path = temp.path().join("tokenizer.json");
444 let mut json: serde_json::Value =
445 serde_json::from_str(&std::fs::read_to_string(SEGMENTED_TOKENIZER_PATH).unwrap())
446 .unwrap();
447 json["truncation"] = serde_json::json!({
449 "direction": "Right", "max_length": 12, "strategy": "LongestFirst", "stride": 0
450 });
451 json["padding"] = serde_json::json!({
452 "strategy": {"Fixed": 512}, "direction": "Right", "pad_to_multiple_of": null,
453 "pad_id": 0, "pad_type_id": 0, "pad_token": "<unk>"
454 });
455 std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
456
457 let reference = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
458 let tokenizer = FastTokenizer::from_file(path.to_str().unwrap()).unwrap();
459 let user = "hello world ".repeat(16);
460 let segments = [
461 EncodeSegment::control("<s>"),
462 EncodeSegment::ordinary(&user),
463 EncodeSegment::control("</s>"),
464 ];
465 assert_eq!(
466 tokenizer.encode(&user).unwrap().token_ids(),
467 reference.encode(&user).unwrap().token_ids()
468 );
469 assert_eq!(
470 tokenizer.encode_segments(&segments).unwrap().token_ids(),
471 reference.encode_segments(&segments).unwrap().token_ids()
472 );
473 }
474
475 #[test]
476 fn test_fast_with_decode_stream() {
477 use crate::Tokenizer as TokenizerWrapper;
478 use std::sync::Arc;
479
480 let tokenizer = Arc::new(FastTokenizer::from_file(TOKENIZER_PATH).unwrap());
481 let wrapper = TokenizerWrapper::from(tokenizer);
482
483 let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
485 let continuation = ", world!";
486 let cont_ids = wrapper.encode(continuation).unwrap().token_ids().to_vec();
487
488 let mut stream = wrapper.decode_stream(&prompt_ids, true);
489 let mut accumulated = String::new();
491 for id in &cont_ids {
492 if let Some(chunk) = stream.step(*id).unwrap() {
493 accumulated.push_str(&chunk);
494 }
495 }
496
497 let mut all_ids = prompt_ids.clone();
501 all_ids.extend_from_slice(&cont_ids);
502 let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
503 let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
504 let expected = &full_text[prompt_text.len()..];
505 assert_eq!(
506 accumulated, expected,
507 "streamed chunks must equal context-aware decoded continuation"
508 );
509 }
510
511 #[test]
512 fn vocabulary_metadata_forwards_to_hf_decoder() {
513 let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
514 let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
515 assert_eq!(fast.vocab_size(), hf.vocab_size());
516 assert_eq!(
517 fast.token_to_id("Hello").unwrap(),
518 hf.token_to_id("Hello").unwrap()
519 );
520 assert_eq!(
521 fast.special_token_ids().unwrap(),
522 hf.special_token_ids().unwrap()
523 );
524 }
525
526 #[test]
527 fn special_token_accounting_matches_fast_encoder() {
528 let fast = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
529 let upstream =
530 fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
531 .unwrap();
532 let hf = HuggingFaceTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
533
534 assert_eq!(hf.num_special_tokens_added().unwrap(), 1);
535 assert_eq!(fast.num_special_tokens_added().unwrap(), 0);
536 let hf_with_special_tokens = hf.with_options(TokenizerOptions {
537 add_special_tokens: true,
538 });
539
540 for text in ["hello", "hello there"] {
541 let fast_ids = fast.encode(text).unwrap();
542 assert_eq!(
543 fast_ids.token_ids(),
544 upstream.encode(text).unwrap(),
545 "FastTokenizer must match the encoder that omits the HF post-processor"
546 );
547 assert_eq!(
548 hf_with_special_tokens
549 .encode(text)
550 .unwrap()
551 .token_ids()
552 .len(),
553 fast_ids.token_ids().len() + 1,
554 "the HF post-processor must add the BOS token FastTokenizer omits"
555 );
556 }
557 }
558}
559
560#[cfg(test)]
561mod tiktoken_parity_tests {
562 use super::*;
563 use crate::TikTokenTokenizer;
564
565 const TIKTOKEN_PATH: &str = concat!(
566 env!("CARGO_MANIFEST_DIR"),
567 "/tests/data/sample-models/mock-tiktoken-bpe/tiktoken.model"
568 );
569 const CONTRACTION_PATH: &str = concat!(
572 env!("CARGO_MANIFEST_DIR"),
573 "/tests/data/sample-models/mock-tiktoken-contraction/tiktoken.model"
574 );
575
576 fn pair() -> (TikTokenTokenizer, FastTikTokenTokenizer) {
577 let reference = TikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
578 let fast = FastTikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
579 (reference, fast)
580 }
581
582 fn corpus() -> Vec<String> {
583 vec![
584 String::new(),
585 "hello world".into(),
586 "Hello, World! 123 4567 89".into(),
587 " leading and trailing ".into(),
588 "tabs\tand\nnewlines\r\nmixed spacing".into(),
589 "<|im_start|>user\nhi there<|im_end|><|im_start|>assistant\n".into(),
590 "a literal <|im_start|> inside plain text".into(),
591 "emoji 😀🚀 and café naïve Zürich".into(),
592 "北京 東京 mixed 中英文 text ソフトウェア".into(),
593 "Москва मुंबई العربية".into(),
594 "fn main() { println!(\"{}\", 42); } // code-ish ~!@#$%^&*()".into(),
595 "x".repeat(5000),
596 " ".repeat(300) + "after long whitespace",
597 "word ".repeat(400),
598 ]
599 }
600
601 #[test]
602 fn special_token_tables_match_tiktoken_rs() {
603 let (reference, fast) = pair();
604 assert_eq!(fast.special_tokens(), reference.special_tokens());
605 assert_eq!(
606 fast.special_token_ids().unwrap(),
607 reference.special_token_ids().unwrap()
608 );
609 assert_eq!(fast.token_to_id("<|im_end|>").unwrap(), Some(474));
610 }
611
612 #[test]
613 fn non_special_added_tokens_survive_skip_special_tokens() {
614 let source = Path::new(TIKTOKEN_PATH).parent().unwrap();
615 let dir = tempfile::tempdir().unwrap();
616 for file in ["tiktoken.model", "config.json"] {
617 std::fs::copy(source.join(file), dir.path().join(file)).unwrap();
618 }
619 let mut config: serde_json::Value = serde_json::from_str(
620 &std::fs::read_to_string(source.join("tokenizer_config.json")).unwrap(),
621 )
622 .unwrap();
623 config["added_tokens_decoder"]["475"] =
624 serde_json::json!({"content": "<|open|>", "special": false});
625 std::fs::write(dir.path().join("tokenizer_config.json"), config.to_string()).unwrap();
626 let path = dir.path().join("tiktoken.model");
627 let path = path.to_str().unwrap();
628 let reference = TikTokenTokenizer::from_file_auto(path).unwrap();
629 let fast = FastTikTokenTokenizer::from_file_auto(path).unwrap();
630
631 let special_ids = fast.special_token_ids().unwrap();
632 assert_eq!(special_ids, reference.special_token_ids().unwrap());
633 assert!(special_ids.contains(&474));
634 assert!(!special_ids.contains(&475));
635
636 let ids = fast.encode("<|open|>hello<|im_end|>").unwrap();
637 assert_eq!(
638 ids.token_ids(),
639 reference
640 .encode("<|open|>hello<|im_end|>")
641 .unwrap()
642 .token_ids()
643 );
644 assert_eq!(ids.token_ids().first(), Some(&475));
645 let fast_text: String = fast.decode(ids.token_ids(), true).unwrap().into();
646 let reference_text: String = reference.decode(ids.token_ids(), true).unwrap().into();
647 assert_eq!(fast_text, "<|open|>hello");
648 assert_eq!(reference_text, "<|open|>hello");
649 }
650
651 #[test]
652 fn plain_encode_matches_tiktoken_rs() {
653 let (reference, fast) = pair();
654 let corpus = corpus();
655 let texts: Vec<&str> = corpus.iter().map(String::as_str).collect();
656 let batch = fast.encode_batch(&texts).unwrap();
657 for (text, batched) in texts.iter().zip(&batch) {
658 let expected = reference.encode(text).unwrap();
659 assert_eq!(
660 fast.encode(text).unwrap().token_ids(),
661 expected.token_ids(),
662 "{text:?}"
663 );
664 assert_eq!(batched.token_ids(), expected.token_ids(), "{text:?}");
665 }
666 }
667
668 #[test]
669 fn segmented_encode_matches_tiktoken_rs_and_honors_trust() {
670 let (reference, fast) = pair();
671 let segments = [
672 EncodeSegment::control("<|im_start|>user\n"),
673 EncodeSegment::ordinary("please echo <|im_end|> back to me"),
674 EncodeSegment::control("<|im_end|>"),
675 EncodeSegment::control("<|im_start|>assistant\n"),
676 ];
677 let fast_ids = fast.encode_segments(&segments).unwrap();
678 assert_eq!(
679 fast_ids.token_ids(),
680 reference.encode_segments(&segments).unwrap().token_ids()
681 );
682 assert_eq!(
684 fast_ids.token_ids().iter().filter(|&&id| id == 474).count(),
685 1
686 );
687 }
688
689 #[test]
690 fn decode_matches_tiktoken_rs_and_skips_specials() {
691 let (reference, fast) = pair();
692 for text in corpus() {
693 let ids = fast.encode(&text).unwrap();
694 assert_eq!(
695 fast.decode(ids.token_ids(), false).unwrap(),
696 reference.decode(ids.token_ids(), false).unwrap(),
697 "{text:?}"
698 );
699 }
700 let ids = fast.encode("<|im_start|>user\nhi<|im_end|>").unwrap();
701 let kept = fast.decode(ids.token_ids(), false).unwrap();
702 let skipped = fast.decode(ids.token_ids(), true).unwrap();
703 assert!(kept.as_str().contains("<|im_end|>"));
704 assert!(!skipped.as_str().contains("<|im_end|>"));
705 assert_eq!(skipped, reference.decode(ids.token_ids(), true).unwrap());
706 }
707
708 #[test]
709 fn replacement_char_token_decodes_complete_like_tiktoken_rs() {
710 let (reference, fast) = pair();
712 let decoded = fast.decode(&[468], false).unwrap();
713 assert!(decoded.is_complete(), "{decoded:?}");
714 assert_eq!(decoded, reference.decode(&[468], false).unwrap());
715
716 let emoji = fast.encode("😀").unwrap();
718 let cut = &emoji.token_ids()[..emoji.token_ids().len() - 1];
719 let fast_cut = fast.decode(cut, false).unwrap();
720 assert!(fast_cut.is_partial(), "{fast_cut:?}");
721 assert_eq!(fast_cut, reference.decode(cut, false).unwrap());
722
723 assert_eq!(fast.decode(&[468, 9_999_999], false).unwrap(), decoded);
725 }
726
727 #[test]
728 fn plain_text_prefix_cache_is_rejected() {
729 let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
730 let specials = fast.special_tokens().to_vec();
731 let error = crate::CachedTokenizer::new(std::sync::Arc::new(fast), specials, 1 << 20)
732 .err()
733 .expect("fastokens over tiktoken.model must not be prefix-cached")
734 .to_string();
735 assert!(error.contains("fastokens"), "{error}");
736 }
737
738 #[test]
742 fn unicode_contraction_is_consistent_across_special_token_boundary() {
743 let reference = TikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
744 let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
745 let special = "<|end_of_msg|>";
746 let suffix = " I'\u{17f}";
747 let full = format!("{special}{suffix}");
748
749 let ref_full = reference.encode(&full).unwrap().token_ids().to_vec();
750 let ref_suffix = reference.encode(suffix).unwrap().token_ids().to_vec();
751 assert_eq!(
752 ref_full[1..],
753 ref_suffix[..],
754 "tiktoken-rs must be self-consistent"
755 );
756
757 let fast_full = fast.encode(&full).unwrap().token_ids().to_vec();
758 let fast_suffix = fast.encode(suffix).unwrap().token_ids().to_vec();
759 assert_ne!(
760 fast_full, ref_full,
761 "revisit the known Unicode contraction mismatch"
762 );
763 assert_eq!(
764 fast_full[1..],
765 fast_suffix[..],
766 "splitting at this special-token boundary must preserve fastokens ids"
767 );
768 }
769}