1use std::collections::HashSet;
14use std::path::Path;
15
16use rayon::prelude::*;
17use rustc_hash::FxHashMap;
18
19use super::{
20 EncodeSegment, Encoding, Error, Result, TokenIdType,
21 hf::HuggingFaceTokenizer,
22 tiktoken,
23 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
24};
25
26fn fast_encode(encoder: &fastokens::Tokenizer, input: &str) -> Result<Encoding> {
27 let ids = encoder
28 .encode(input)
29 .map_err(|e| Error::msg(format!("Fastokens encode error: {e}")))?;
30 Ok(Encoding::Sp(ids))
31}
32
33fn fast_encode_segments(
34 encoder: &fastokens::Tokenizer,
35 segments: &[EncodeSegment<'_>],
36) -> Result<Encoding> {
37 let segments: Vec<fastokens::EncodeSegment<'_>> = segments
38 .iter()
39 .map(|segment| fastokens::EncodeSegment {
40 text: segment.text,
41 allow_special: segment.allow_special,
42 })
43 .collect();
44 let ids = encoder
45 .encode_segments(&segments)
46 .map_err(|e| Error::msg(format!("Fastokens segmented encode error: {e}")))?;
47 Ok(Encoding::Sp(ids))
48}
49
50pub struct FastTokenizer {
54 fast_encoder: fastokens::Tokenizer,
55 hf_decoder: HuggingFaceTokenizer,
56}
57
58impl FastTokenizer {
59 pub fn from_file(path: &str) -> Result<Self> {
64 let mut fast_encoder = fastokens::Tokenizer::from_file(Path::new(path))
65 .map_err(|e| Error::msg(format!("Error loading fastokens tokenizer: {e}")))?;
66 fast_encoder.set_truncation(None);
67 fast_encoder.set_padding(None);
68 let hf_decoder = HuggingFaceTokenizer::from_file(path)?;
69 Ok(Self {
70 fast_encoder,
71 hf_decoder,
72 })
73 }
74}
75
76impl Encoder for FastTokenizer {
77 fn encode(&self, input: &str) -> Result<Encoding> {
78 fast_encode(&self.fast_encoder, input)
79 }
80
81 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
82 inputs.par_iter().map(|input| self.encode(input)).collect()
83 }
84
85 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
86 fast_encode_segments(&self.fast_encoder, segments)
87 }
88}
89
90impl Decoder for FastTokenizer {
91 fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
92 self.hf_decoder
93 .has_unstable_suffix(token_ids, skip_special_tokens)
94 }
95
96 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
97 self.hf_decoder.decode(token_ids, skip_special_tokens)
98 }
99}
100
101impl Tokenizer for FastTokenizer {
102 fn validate_prefix_cache(&self) -> Result<()> {
103 Ok(())
104 }
105
106 fn vocab_size(&self) -> Option<usize> {
109 self.hf_decoder.vocab_size()
110 }
111
112 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
113 self.hf_decoder.token_to_id(token)
114 }
115
116 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
117 self.hf_decoder.special_token_ids()
118 }
119
120 fn num_special_tokens_added(&self) -> Result<usize> {
121 Ok(0)
122 }
123}
124
125pub struct FastTikTokenTokenizer {
135 inner: fastokens::Tokenizer,
136 id_to_bytes: FxHashMap<u32, Vec<u8>>,
139 special_token_ids: HashSet<u32>,
140 special_tokens: Vec<String>,
141}
142
143impl FastTikTokenTokenizer {
144 pub fn from_file_auto(path: &str) -> Result<Self> {
147 let directory = Path::new(path)
148 .parent()
149 .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
150 let pattern = tiktoken::detect_bpe_pattern(directory)?;
151 let encoder = tiktoken::parse_tiktoken_file(path)?;
152 let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
153 let tiktoken::AddedTokens { tokens, skip_ids } =
154 tiktoken::load_added_tokens(directory, num_base_tokens)?;
155
156 let ranks: Vec<(Vec<u8>, u32)> = encoder.into_iter().collect();
157 let mut id_to_bytes: FxHashMap<u32, Vec<u8>> = ranks
158 .iter()
159 .map(|(bytes, rank)| (*rank, bytes.clone()))
160 .collect();
161 id_to_bytes.extend(
162 tokens
163 .iter()
164 .map(|(content, &id)| (id, content.as_bytes().to_vec())),
165 );
166 let special_token_strings = tiktoken::sorted_special_token_strings(&tokens);
167
168 let config =
169 fastokens::tiktoken::TiktokenConfig::new(pattern, tokens.into_iter().collect());
170 let inner = fastokens::Tokenizer::from_tiktoken_ranks(&ranks, config).map_err(|e| {
171 Error::msg(format!(
172 "Error loading fastokens tiktoken tokenizer from {path}: {e}"
173 ))
174 })?;
175 Ok(Self {
176 inner,
177 id_to_bytes,
178 special_token_ids: skip_ids,
179 special_tokens: special_token_strings,
180 })
181 }
182
183 pub fn special_tokens(&self) -> &[String] {
187 &self.special_tokens
188 }
189}
190
191impl Encoder for FastTikTokenTokenizer {
192 fn encode(&self, input: &str) -> Result<Encoding> {
193 fast_encode(&self.inner, input)
194 }
195
196 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
197 inputs.par_iter().map(|input| self.encode(input)).collect()
198 }
199
200 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
201 fast_encode_segments(&self.inner, segments)
202 }
203}
204
205impl Decoder for FastTikTokenTokenizer {
206 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
207 let mut bytes = Vec::new();
208 for id in token_ids {
209 if skip_special_tokens && self.special_token_ids.contains(id) {
210 continue;
211 }
212 if let Some(token) = self.id_to_bytes.get(id) {
213 bytes.extend_from_slice(token);
214 }
215 }
216 match String::from_utf8(bytes) {
217 Ok(text) => Ok(DecodeResult::Complete(text)),
218 Err(e) => Ok(DecodeResult::from_decoded(
219 String::from_utf8_lossy(e.as_bytes()).into_owned(),
220 )),
221 }
222 }
223}
224
225impl Tokenizer for FastTikTokenTokenizer {
226 fn validate_prefix_cache(&self) -> Result<()> {
227 Err(Error::msg(
228 "fastokens over tiktoken.model plain-text prefix caching is disabled pending \
229 validation of the full cache matrix",
230 ))
231 }
232
233 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
234 Ok(self.inner.token_to_id(token))
235 }
236
237 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
238 let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
239 ids.sort_unstable();
240 Ok(ids)
241 }
242
243 fn num_special_tokens_added(&self) -> Result<usize> {
244 Ok(0)
245 }
246}
247
248#[cfg(test)]
249mod tests {
250 use super::*;
251 use crate::{HuggingFaceTokenizer, TokenizerOptions};
252
253 const TOKENIZER_PATH: &str = concat!(
256 env!("CARGO_MANIFEST_DIR"),
257 "/tests/data/minimal-bpe/tokenizer.json"
258 );
259 const SEGMENTED_TOKENIZER_PATH: &str = concat!(
260 env!("CARGO_MANIFEST_DIR"),
261 "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
262 );
263
264 #[test]
265 fn byte_fallback_stream_matches_full_decode() {
266 let tokenizer: crate::Tokenizer =
267 std::sync::Arc::new(FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap()).into();
268 for (pieces, skip) in [
269 (vec!["<0x61>", "<0xF5>"], false),
270 (vec!["<0x61>", "</s>", "<0xF5>"], false),
271 (vec!["<0x61>", "</s>", "<0xF5>"], true),
272 ] {
273 let ids: Vec<_> = pieces
274 .iter()
275 .map(|piece| tokenizer.token_to_id(piece).unwrap().unwrap())
276 .collect();
277 let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
278 let mut stream = tokenizer.decode_stream(&[], skip);
279 let mut actual = String::new();
280 for id in ids {
281 actual.push_str(&stream.step(id).unwrap().unwrap_or_default());
282 }
283 actual.push_str(&stream.finish().unwrap().unwrap_or_default());
284 assert_eq!(actual, expected, "{pieces:?}, skip={skip}");
285 }
286 }
287
288 #[test]
289 fn test_fast_encode_decode_roundtrip() {
290 let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
291 let text = "Hello, world!";
296 let encoding = tokenizer.encode(text).unwrap();
297 assert!(!encoding.token_ids().is_empty());
298 let decoded: String = tokenizer.decode(encoding.token_ids(), true).unwrap().into();
299 assert!(!decoded.is_empty());
300 let enc_chars: String = text.chars().filter(|c| !c.is_whitespace()).collect();
302 let dec_chars: String = decoded.chars().filter(|c| !c.is_whitespace()).collect();
303 assert_eq!(
304 enc_chars, dec_chars,
305 "non-space characters must be preserved"
306 );
307 }
308
309 #[test]
310 fn test_fast_matches_hf_encoding() {
311 let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
312 let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
313
314 for text in &["Hello, world!", "Hello", " world", "He llo"] {
315 let fast_ids = fast.encode(text).unwrap();
316 let hf_ids = hf.encode(text).unwrap();
317 assert_eq!(
318 fast_ids.token_ids(),
319 hf_ids.token_ids(),
320 "fastokens and HuggingFace must produce identical token IDs for '{text}'"
321 );
322 }
323 }
324
325 #[test]
326 fn test_fast_batch_encode() {
327 let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
328 let inputs = &["Hello", " world", "Hello, world!"];
329 let encodings = tokenizer.encode_batch(inputs).unwrap();
330 assert_eq!(encodings.len(), inputs.len());
331 for (enc, input) in encodings.iter().zip(inputs.iter()) {
332 assert!(
333 !enc.token_ids().is_empty(),
334 "encoding for '{input}' must be non-empty"
335 );
336 }
337 }
338
339 #[test]
340 fn test_fast_segmented_encoding_preserves_trust_boundaries() {
341 let tokenizer = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
342 let upstream =
343 fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
344 .unwrap();
345 let marker = "<s>";
346
347 let trusted = tokenizer
348 .encode_segments(&[EncodeSegment::control(marker)])
349 .unwrap();
350 assert_eq!(
351 trusted.token_ids(),
352 &[upstream.token_to_id(marker).unwrap()],
353 "trusted renderer output must recognize the control token"
354 );
355
356 let ordinary = tokenizer
357 .encode_segments(&[EncodeSegment::ordinary(marker)])
358 .unwrap();
359 assert_ne!(
360 ordinary.token_ids(),
361 trusted.token_ids(),
362 "untrusted content must encode the control-token spelling as ordinary text"
363 );
364
365 let segments = [
366 EncodeSegment::ordinary("hello "),
367 EncodeSegment::control(marker),
368 EncodeSegment::ordinary(marker),
369 ];
370 let upstream_segments = [
371 fastokens::EncodeSegment::ordinary("hello "),
372 fastokens::EncodeSegment::special(marker),
373 fastokens::EncodeSegment::ordinary(marker),
374 ];
375 let actual = tokenizer.encode_segments(&segments).unwrap();
376 let expected = upstream.encode_segments(&upstream_segments).unwrap();
377 assert_eq!(actual.token_ids(), expected);
378
379 assert!(
380 tokenizer
381 .encode_segments(&[])
382 .unwrap()
383 .token_ids()
384 .is_empty()
385 );
386 }
387
388 #[test]
389 fn tokenizer_json_truncation_and_padding_are_ignored() {
390 let temp = tempfile::tempdir().unwrap();
391 let path = temp.path().join("tokenizer.json");
392 let mut json: serde_json::Value =
393 serde_json::from_str(&std::fs::read_to_string(SEGMENTED_TOKENIZER_PATH).unwrap())
394 .unwrap();
395 json["truncation"] = serde_json::json!({
397 "direction": "Right", "max_length": 12, "strategy": "LongestFirst", "stride": 0
398 });
399 json["padding"] = serde_json::json!({
400 "strategy": {"Fixed": 512}, "direction": "Right", "pad_to_multiple_of": null,
401 "pad_id": 0, "pad_type_id": 0, "pad_token": "<unk>"
402 });
403 std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
404
405 let reference = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
406 let tokenizer = FastTokenizer::from_file(path.to_str().unwrap()).unwrap();
407 let user = "hello world ".repeat(16);
408 let segments = [
409 EncodeSegment::control("<s>"),
410 EncodeSegment::ordinary(&user),
411 EncodeSegment::control("</s>"),
412 ];
413 assert_eq!(
414 tokenizer.encode(&user).unwrap().token_ids(),
415 reference.encode(&user).unwrap().token_ids()
416 );
417 assert_eq!(
418 tokenizer.encode_segments(&segments).unwrap().token_ids(),
419 reference.encode_segments(&segments).unwrap().token_ids()
420 );
421 }
422
423 #[test]
424 fn test_fast_with_decode_stream() {
425 use crate::Tokenizer as TokenizerWrapper;
426 use std::sync::Arc;
427
428 let tokenizer = Arc::new(FastTokenizer::from_file(TOKENIZER_PATH).unwrap());
429 let wrapper = TokenizerWrapper::from(tokenizer);
430
431 let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
433 let continuation = ", world!";
434 let cont_ids = wrapper.encode(continuation).unwrap().token_ids().to_vec();
435
436 let mut stream = wrapper.decode_stream(&prompt_ids, true);
437 let mut accumulated = String::new();
439 for id in &cont_ids {
440 if let Some(chunk) = stream.step(*id).unwrap() {
441 accumulated.push_str(&chunk);
442 }
443 }
444
445 let mut all_ids = prompt_ids.clone();
449 all_ids.extend_from_slice(&cont_ids);
450 let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
451 let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
452 let expected = &full_text[prompt_text.len()..];
453 assert_eq!(
454 accumulated, expected,
455 "streamed chunks must equal context-aware decoded continuation"
456 );
457 }
458
459 #[test]
460 fn vocabulary_metadata_forwards_to_hf_decoder() {
461 let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
462 let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
463 assert_eq!(fast.vocab_size(), hf.vocab_size());
464 assert_eq!(
465 fast.token_to_id("Hello").unwrap(),
466 hf.token_to_id("Hello").unwrap()
467 );
468 assert_eq!(
469 fast.special_token_ids().unwrap(),
470 hf.special_token_ids().unwrap()
471 );
472 }
473
474 #[test]
475 fn special_token_accounting_matches_fast_encoder() {
476 let fast = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
477 let upstream =
478 fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
479 .unwrap();
480 let hf = HuggingFaceTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
481
482 assert_eq!(hf.num_special_tokens_added().unwrap(), 1);
483 assert_eq!(fast.num_special_tokens_added().unwrap(), 0);
484 let hf_with_special_tokens = hf.with_options(TokenizerOptions {
485 add_special_tokens: true,
486 });
487
488 for text in ["hello", "hello there"] {
489 let fast_ids = fast.encode(text).unwrap();
490 assert_eq!(
491 fast_ids.token_ids(),
492 upstream.encode(text).unwrap(),
493 "FastTokenizer must match the encoder that omits the HF post-processor"
494 );
495 assert_eq!(
496 hf_with_special_tokens
497 .encode(text)
498 .unwrap()
499 .token_ids()
500 .len(),
501 fast_ids.token_ids().len() + 1,
502 "the HF post-processor must add the BOS token FastTokenizer omits"
503 );
504 }
505 }
506}
507
508#[cfg(test)]
509mod tiktoken_parity_tests {
510 use super::*;
511 use crate::TikTokenTokenizer;
512
513 const TIKTOKEN_PATH: &str = concat!(
514 env!("CARGO_MANIFEST_DIR"),
515 "/tests/data/sample-models/mock-tiktoken-bpe/tiktoken.model"
516 );
517 const CONTRACTION_PATH: &str = concat!(
520 env!("CARGO_MANIFEST_DIR"),
521 "/tests/data/sample-models/mock-tiktoken-contraction/tiktoken.model"
522 );
523
524 fn pair() -> (TikTokenTokenizer, FastTikTokenTokenizer) {
525 let reference = TikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
526 let fast = FastTikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
527 (reference, fast)
528 }
529
530 fn corpus() -> Vec<String> {
531 vec![
532 String::new(),
533 "hello world".into(),
534 "Hello, World! 123 4567 89".into(),
535 " leading and trailing ".into(),
536 "tabs\tand\nnewlines\r\nmixed spacing".into(),
537 "<|im_start|>user\nhi there<|im_end|><|im_start|>assistant\n".into(),
538 "a literal <|im_start|> inside plain text".into(),
539 "emoji 😀🚀 and café naïve Zürich".into(),
540 "北京 東京 mixed 中英文 text ソフトウェア".into(),
541 "Москва मुंबई العربية".into(),
542 "fn main() { println!(\"{}\", 42); } // code-ish ~!@#$%^&*()".into(),
543 "x".repeat(5000),
544 " ".repeat(300) + "after long whitespace",
545 "word ".repeat(400),
546 ]
547 }
548
549 #[test]
550 fn special_token_tables_match_tiktoken_rs() {
551 let (reference, fast) = pair();
552 assert_eq!(fast.special_tokens(), reference.special_tokens());
553 assert_eq!(
554 fast.special_token_ids().unwrap(),
555 reference.special_token_ids().unwrap()
556 );
557 assert_eq!(fast.token_to_id("<|im_end|>").unwrap(), Some(474));
558 }
559
560 #[test]
561 fn non_special_added_tokens_survive_skip_special_tokens() {
562 let source = Path::new(TIKTOKEN_PATH).parent().unwrap();
563 let dir = tempfile::tempdir().unwrap();
564 for file in ["tiktoken.model", "config.json"] {
565 std::fs::copy(source.join(file), dir.path().join(file)).unwrap();
566 }
567 let mut config: serde_json::Value = serde_json::from_str(
568 &std::fs::read_to_string(source.join("tokenizer_config.json")).unwrap(),
569 )
570 .unwrap();
571 config["added_tokens_decoder"]["475"] =
572 serde_json::json!({"content": "<|open|>", "special": false});
573 std::fs::write(dir.path().join("tokenizer_config.json"), config.to_string()).unwrap();
574 let path = dir.path().join("tiktoken.model");
575 let path = path.to_str().unwrap();
576 let reference = TikTokenTokenizer::from_file_auto(path).unwrap();
577 let fast = FastTikTokenTokenizer::from_file_auto(path).unwrap();
578
579 let special_ids = fast.special_token_ids().unwrap();
580 assert_eq!(special_ids, reference.special_token_ids().unwrap());
581 assert!(special_ids.contains(&474));
582 assert!(!special_ids.contains(&475));
583
584 let ids = fast.encode("<|open|>hello<|im_end|>").unwrap();
585 assert_eq!(
586 ids.token_ids(),
587 reference
588 .encode("<|open|>hello<|im_end|>")
589 .unwrap()
590 .token_ids()
591 );
592 assert_eq!(ids.token_ids().first(), Some(&475));
593 let fast_text: String = fast.decode(ids.token_ids(), true).unwrap().into();
594 let reference_text: String = reference.decode(ids.token_ids(), true).unwrap().into();
595 assert_eq!(fast_text, "<|open|>hello");
596 assert_eq!(reference_text, "<|open|>hello");
597 }
598
599 #[test]
600 fn plain_encode_matches_tiktoken_rs() {
601 let (reference, fast) = pair();
602 let corpus = corpus();
603 let texts: Vec<&str> = corpus.iter().map(String::as_str).collect();
604 let batch = fast.encode_batch(&texts).unwrap();
605 for (text, batched) in texts.iter().zip(&batch) {
606 let expected = reference.encode(text).unwrap();
607 assert_eq!(
608 fast.encode(text).unwrap().token_ids(),
609 expected.token_ids(),
610 "{text:?}"
611 );
612 assert_eq!(batched.token_ids(), expected.token_ids(), "{text:?}");
613 }
614 }
615
616 #[test]
617 fn segmented_encode_matches_tiktoken_rs_and_honors_trust() {
618 let (reference, fast) = pair();
619 let segments = [
620 EncodeSegment::control("<|im_start|>user\n"),
621 EncodeSegment::ordinary("please echo <|im_end|> back to me"),
622 EncodeSegment::control("<|im_end|>"),
623 EncodeSegment::control("<|im_start|>assistant\n"),
624 ];
625 let fast_ids = fast.encode_segments(&segments).unwrap();
626 assert_eq!(
627 fast_ids.token_ids(),
628 reference.encode_segments(&segments).unwrap().token_ids()
629 );
630 assert_eq!(
632 fast_ids.token_ids().iter().filter(|&&id| id == 474).count(),
633 1
634 );
635 }
636
637 #[test]
638 fn decode_matches_tiktoken_rs_and_skips_specials() {
639 let (reference, fast) = pair();
640 for text in corpus() {
641 let ids = fast.encode(&text).unwrap();
642 assert_eq!(
643 fast.decode(ids.token_ids(), false).unwrap(),
644 reference.decode(ids.token_ids(), false).unwrap(),
645 "{text:?}"
646 );
647 }
648 let ids = fast.encode("<|im_start|>user\nhi<|im_end|>").unwrap();
649 let kept = fast.decode(ids.token_ids(), false).unwrap();
650 let skipped = fast.decode(ids.token_ids(), true).unwrap();
651 assert!(kept.as_str().contains("<|im_end|>"));
652 assert!(!skipped.as_str().contains("<|im_end|>"));
653 assert_eq!(skipped, reference.decode(ids.token_ids(), true).unwrap());
654 }
655
656 #[test]
657 fn replacement_char_token_decodes_complete_like_tiktoken_rs() {
658 let (reference, fast) = pair();
660 let decoded = fast.decode(&[468], false).unwrap();
661 assert!(decoded.is_complete(), "{decoded:?}");
662 assert_eq!(decoded, reference.decode(&[468], false).unwrap());
663
664 let emoji = fast.encode("😀").unwrap();
666 let cut = &emoji.token_ids()[..emoji.token_ids().len() - 1];
667 let fast_cut = fast.decode(cut, false).unwrap();
668 assert!(fast_cut.is_partial(), "{fast_cut:?}");
669 assert_eq!(fast_cut, reference.decode(cut, false).unwrap());
670
671 assert_eq!(fast.decode(&[468, 9_999_999], false).unwrap(), decoded);
673 }
674
675 #[test]
676 fn plain_text_prefix_cache_is_rejected() {
677 let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
678 let specials = fast.special_tokens().to_vec();
679 let error = crate::CachedTokenizer::new(std::sync::Arc::new(fast), specials, 1 << 20)
680 .err()
681 .expect("fastokens over tiktoken.model must not be prefix-cached")
682 .to_string();
683 assert!(error.contains("fastokens"), "{error}");
684 }
685
686 #[test]
690 fn unicode_contraction_is_consistent_across_special_token_boundary() {
691 let reference = TikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
692 let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
693 let special = "<|end_of_msg|>";
694 let suffix = " I'\u{17f}";
695 let full = format!("{special}{suffix}");
696
697 let ref_full = reference.encode(&full).unwrap().token_ids().to_vec();
698 let ref_suffix = reference.encode(suffix).unwrap().token_ids().to_vec();
699 assert_eq!(
700 ref_full[1..],
701 ref_suffix[..],
702 "tiktoken-rs must be self-consistent"
703 );
704
705 let fast_full = fast.encode(&full).unwrap().token_ids().to_vec();
706 let fast_suffix = fast.encode(suffix).unwrap().token_ids().to_vec();
707 assert_ne!(
708 fast_full, ref_full,
709 "revisit the known Unicode contraction mismatch"
710 );
711 assert_eq!(
712 fast_full[1..],
713 fast_suffix[..],
714 "splitting at this special-token boundary must preserve fastokens ids"
715 );
716 }
717}