1mod l1;
74
75use std::sync::Arc;
76
77use l1::PrefixLookup;
78pub use l1::{
79 CacheEventFn, L1Cache, L1CacheStats, SharedTokenizerCache, SharedTokenizerCacheStats,
80};
81
82use crate::{
83 EncodeSegment, Encoding, Result, TokenIdType,
84 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
85};
86
87const MIN_CACHED_SEGMENT_BYTES: usize = 128;
90
91#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96pub struct CacheTokenUsage {
97 pub cached_tokens: usize,
99 pub uncached_tokens: usize,
101}
102
103pub type CacheTokenUsageFn = Arc<dyn Fn(CacheTokenUsage) + Send + Sync>;
105
106pub struct CachedTokenizer {
111 inner: Arc<dyn Tokenizer>,
112 l1: L1Cache,
113 l1_enabled: bool,
114 extend_on_hit: bool,
115 token_observer: Option<CacheTokenUsageFn>,
117}
118
119impl CachedTokenizer {
120 pub fn new(
137 inner: Arc<dyn Tokenizer>,
138 special_tokens: Vec<String>,
139 max_memory_bytes: usize,
140 ) -> Result<Self> {
141 Self::build(inner, special_tokens, |tokens| {
142 L1Cache::new(max_memory_bytes, tokens)
143 })
144 }
145
146 pub fn new_with_cache(
158 inner: Arc<dyn Tokenizer>,
159 special_tokens: Vec<String>,
160 shared_cache: SharedTokenizerCache,
161 namespace: &[u8],
162 ) -> Result<Self> {
163 Self::build(inner, special_tokens, |tokens| {
164 L1Cache::new_with_cache(shared_cache, tokens, namespace)
165 })
166 }
167
168 fn build(
169 inner: Arc<dyn Tokenizer>,
170 mut special_tokens: Vec<String>,
171 make_cache: impl FnOnce(Vec<String>) -> L1Cache,
172 ) -> Result<Self> {
173 inner.validate_prefix_cache()?;
174 special_tokens.retain(|token| !token.is_empty());
175
176 let overlapping_specials = match l1::first_unsafe_overlap(&special_tokens) {
179 Some((first, second)) => {
180 tracing::warn!(
181 target: "tokenizer",
182 first_token = first,
183 second_token = second,
184 special_token_count = special_tokens.len(),
185 "special tokens can overlap; tokenizer prefix cache disabled"
186 );
187 true
188 }
189 None => false,
190 };
191
192 let l1_enabled = !special_tokens.is_empty() && !overlapping_specials;
193 let cache_tokens = if l1_enabled {
194 special_tokens
195 } else {
196 Vec::new()
197 };
198 Ok(Self {
199 inner,
200 l1: make_cache(cache_tokens),
201 l1_enabled,
202 extend_on_hit: false,
203 token_observer: None,
204 })
205 }
206
207 pub fn with_extend(mut self, enabled: bool) -> Self {
212 self.extend_on_hit = enabled;
213 self
214 }
215
216 pub fn with_observer(mut self, on_hit: CacheEventFn, on_miss: CacheEventFn) -> Self {
220 self.l1.set_observer(on_hit, on_miss);
221 self
222 }
223
224 pub fn with_token_observer(mut self, observer: CacheTokenUsageFn) -> Self {
232 self.token_observer = Some(observer);
233 self
234 }
235
236 fn observe_token_usage(&self, cached_tokens: usize, total_tokens: usize) {
237 if let Some(observer) = &self.token_observer {
238 let uncached_tokens = total_tokens
239 .checked_sub(cached_tokens)
240 .expect("cached token count cannot exceed total token count");
241 observer(CacheTokenUsage {
242 cached_tokens,
243 uncached_tokens,
244 });
245 }
246 }
247
248 pub fn cache_stats(&self) -> L1CacheStats {
252 if self.l1_enabled {
253 self.l1.stats()
254 } else {
255 L1CacheStats::default()
256 }
257 }
258
259 pub fn inner(&self) -> &Arc<dyn Tokenizer> {
261 &self.inner
262 }
263}
264
265impl Encoder for CachedTokenizer {
266 fn encode(&self, input: &str) -> Result<Encoding> {
267 if !self.l1_enabled {
268 return self.inner.encode(input);
269 }
270
271 let matched = match self.l1.lookup_prefix(input) {
272 PrefixLookup::Hit(matched) => matched,
273 PrefixLookup::Miss(prefix_hashes) => {
274 let encoding = Encoding::Sp(self.l1.populate_and_encode_with_hashes(
275 input,
276 prefix_hashes.into_iter(),
277 self.inner.as_ref(),
278 )?);
279 self.observe_token_usage(0, encoding.token_ids().len());
280 return Ok(encoding);
281 }
282 };
283
284 let cached_tokens = matched.tokens.len();
285 let encoding = if self.extend_on_hit {
286 Encoding::Sp(self.l1.extend_after_match_with_hash(
287 input,
288 matched,
289 self.inner.as_ref(),
290 )?)
291 } else {
292 let suffix_enc = self.inner.encode(&input[matched.prefix_len..])?;
293 let mut merged: Vec<TokenIdType> =
295 Vec::with_capacity(matched.tokens.len() + suffix_enc.token_ids().len());
296 merged.extend_from_slice(&matched.tokens);
297 merged.extend_from_slice(suffix_enc.token_ids());
298 Encoding::Sp(merged)
299 };
300 self.observe_token_usage(cached_tokens, encoding.token_ids().len());
301 Ok(encoding)
302 }
303
304 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
305 if !self.l1_enabled {
309 return self.inner.encode_batch(inputs);
310 }
311
312 inputs.iter().map(|&i| self.encode(i)).collect()
316 }
317
318 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
319 if !self.l1_enabled {
320 return self.inner.encode_segments(segments);
321 }
322
323 let mut ids: Vec<TokenIdType> = Vec::new();
328 let mut cached_tokens = 0;
329 let mut looked_up = false;
330 let mut hit = false;
331 let mut run_start = 0;
332 for (index, segment) in segments.iter().enumerate() {
333 if segment.text.len() < MIN_CACHED_SEGMENT_BYTES {
334 continue;
335 }
336 if run_start < index {
337 let run = self.inner.encode_segments(&segments[run_start..index])?;
338 ids.extend_from_slice(run.token_ids());
339 }
340 run_start = index + 1;
341 looked_up = true;
342 let key = self.l1.segment_key(segment.text, segment.allow_special);
343 if let Some(tokens) = self.l1.get(&key) {
344 hit = true;
345 cached_tokens += tokens.len();
346 ids.extend_from_slice(&tokens);
347 } else {
348 let encoding = self.inner.encode_segments(std::slice::from_ref(segment))?;
349 let tokens: Arc<[TokenIdType]> = encoding.token_ids().into();
350 ids.extend_from_slice(&tokens);
351 self.l1.insert(key, tokens);
352 }
353 }
354 if run_start < segments.len() {
355 let run = self.inner.encode_segments(&segments[run_start..])?;
356 ids.extend_from_slice(run.token_ids());
357 }
358 if looked_up {
359 self.l1.record_lookup(hit);
360 }
361 self.observe_token_usage(cached_tokens, ids.len());
362 Ok(Encoding::Sp(ids))
363 }
364}
365
366impl Decoder for CachedTokenizer {
367 fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
368 self.inner
369 .has_unstable_suffix(token_ids, skip_special_tokens)
370 }
371
372 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
373 self.inner.decode(token_ids, skip_special_tokens)
375 }
376}
377
378impl Tokenizer for CachedTokenizer {
379 fn vocab_size(&self) -> Option<usize> {
380 self.inner.vocab_size()
381 }
382
383 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
384 self.inner.token_to_id(token)
385 }
386
387 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
388 self.inner.special_token_ids()
389 }
390
391 fn num_special_tokens_added(&self) -> Result<usize> {
392 Ok(0)
393 }
394}
395
396#[cfg(test)]
397mod tests {
398 use super::*;
399 use crate::HuggingFaceTokenizer;
400 use std::sync::{Mutex, atomic::AtomicU64, atomic::Ordering};
401 use tokenizers::Tokenizer as HfTokenizer;
402
403 struct FailingTokenizer;
404
405 struct SegmentTokenizer;
406
407 impl Encoder for SegmentTokenizer {
408 fn encode(&self, input: &str) -> Result<Encoding> {
409 Ok(Encoding::Sp(vec![input.len() as u32]))
410 }
411
412 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
413 inputs.iter().map(|input| self.encode(input)).collect()
414 }
415
416 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
417 let ids = segments
418 .iter()
419 .flat_map(|segment| [segment.allow_special as u32, segment.text.len() as u32])
420 .collect();
421 Ok(Encoding::Sp(ids))
422 }
423 }
424
425 impl Decoder for SegmentTokenizer {
426 fn decode(
427 &self,
428 _token_ids: &[TokenIdType],
429 _skip_special_tokens: bool,
430 ) -> Result<DecodeResult> {
431 Ok(DecodeResult::Complete(String::new()))
432 }
433 }
434
435 impl Tokenizer for SegmentTokenizer {
436 fn validate_prefix_cache(&self) -> Result<()> {
437 Ok(())
438 }
439 }
440
441 impl Encoder for FailingTokenizer {
442 fn encode(&self, _input: &str) -> Result<Encoding> {
443 Err(anyhow::anyhow!("intentional encode failure"))
444 }
445
446 fn encode_batch(&self, _inputs: &[&str]) -> Result<Vec<Encoding>> {
447 Err(anyhow::anyhow!("intentional encode failure"))
448 }
449 }
450
451 impl Decoder for FailingTokenizer {
452 fn decode(
453 &self,
454 _token_ids: &[TokenIdType],
455 _skip_special_tokens: bool,
456 ) -> Result<DecodeResult> {
457 Err(anyhow::anyhow!("intentional decode failure"))
458 }
459 }
460
461 impl Tokenizer for FailingTokenizer {
462 fn validate_prefix_cache(&self) -> Result<()> {
463 Ok(())
464 }
465
466 fn vocab_size(&self) -> Option<usize> {
467 None
468 }
469 }
470
471 const TINYLLAMA_PATH: &str = concat!(
472 env!("CARGO_MANIFEST_DIR"),
473 "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
474 );
475
476 fn inner() -> Arc<dyn Tokenizer> {
477 Arc::new(HuggingFaceTokenizer::from_file(TINYLLAMA_PATH).expect("load TinyLlama"))
478 }
479
480 fn specials() -> Vec<String> {
481 vec!["<s>".into(), "</s>".into()]
482 }
483
484 fn collect_token_usage(
485 tokenizer: CachedTokenizer,
486 ) -> (CachedTokenizer, Arc<Mutex<Vec<CacheTokenUsage>>>) {
487 let events = Arc::new(Mutex::new(Vec::new()));
488 let observed = events.clone();
489 let tokenizer = tokenizer.with_token_observer(Arc::new(move |usage| {
490 observed.lock().unwrap().push(usage);
491 }));
492 (tokenizer, events)
493 }
494
495 #[test]
496 fn rejects_hf_tokenizer_that_adds_special_tokens() {
497 let tokenizer: Arc<dyn Tokenizer> = Arc::new(
498 HuggingFaceTokenizer::from_file(TINYLLAMA_PATH)
499 .expect("load TinyLlama")
500 .with_options(crate::TokenizerOptions {
501 add_special_tokens: true,
502 }),
503 );
504
505 let result = CachedTokenizer::new(tokenizer, specials(), 4096);
506 let Err(error) = result else {
507 panic!("add_special_tokens=true must be rejected");
508 };
509 assert_eq!(
510 error.to_string(),
511 "HuggingFace tokenizers configured with add_special_tokens=true must remain uncached"
512 );
513 }
514
515 #[test]
516 fn empty_specials_passes_through_correctly() {
517 let tok = inner();
519 let (cached, events) = collect_token_usage(
520 CachedTokenizer::new(tok.clone(), vec![String::new()], 4096)
521 .expect("TinyLlama must support prefix caching"),
522 );
523 let s = "<s>hello world</s>";
524 let a = cached.encode(s).unwrap();
525 let b = tok.encode(s).unwrap();
526 assert_eq!(a.token_ids(), b.token_ids());
527 let stats = cached.cache_stats();
528 assert_eq!(stats.entries, 0);
529 assert_eq!(stats.misses, 0, "empty specials must not increment misses");
530 assert_eq!(stats.hits, 0);
531 assert!(
532 events.lock().unwrap().is_empty(),
533 "empty specials must not emit token usage"
534 );
535 }
536
537 #[test]
538 fn laguna_overlapping_specials_bypass_cache() {
539 const TOKENIZER_JSON: &str = r#"{
540 "version": "1.0",
541 "truncation": null,
542 "padding": null,
543 "added_tokens": [
544 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
545 {"id": 2, "content": "〈|EOS|〉", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
546 {"id": 14, "content": "〈|", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
547 {"id": 15, "content": "|〉", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
548 ],
549 "normalizer": null,
550 "pre_tokenizer": null,
551 "post_processor": null,
552 "decoder": null,
553 "model": {
554 "type": "WordLevel",
555 "vocab": {"<unk>": 0, "〈|EOS|〉": 2, "〈|": 14, "|〉": 15, "tail": 16},
556 "unk_token": "<unk>"
557 }
558 }"#;
559
560 let hf = HfTokenizer::from_bytes(TOKENIZER_JSON).expect("load test tokenizer");
561 let tok: Arc<dyn Tokenizer> = Arc::new(HuggingFaceTokenizer::from_tokenizer(hf));
562 let overlapping = vec!["〈|EOS|〉".into(), "〈|".into(), "|〉".into()];
563 let (cached, events) = collect_token_usage(
564 CachedTokenizer::new(tok.clone(), overlapping, 4096)
565 .expect("HuggingFace tokenizer must support prefix caching"),
566 );
567
568 let expected = tok.encode("〈|EOS|〉").unwrap();
569 assert_eq!(expected.token_ids(), &[2]);
570 assert_eq!(
571 cached.encode("〈|EOS|〉").unwrap().token_ids(),
572 expected.token_ids()
573 );
574 let stats = cached.cache_stats();
575 assert_eq!(stats.entries, 0);
576 assert_eq!(
577 stats.misses, 0,
578 "overlapping specials must not increment misses"
579 );
580 assert_eq!(stats.hits, 0);
581 assert!(
582 events.lock().unwrap().is_empty(),
583 "overlapping specials must not emit token usage"
584 );
585 }
586
587 fn long_text(text: &str) -> String {
589 text.repeat(MIN_CACHED_SEGMENT_BYTES / text.len() + 1)
590 }
591
592 #[test]
593 fn segmented_encoding_without_specials_passes_through() {
594 let inner: Arc<dyn Tokenizer> = Arc::new(SegmentTokenizer);
595 let user = long_text("user content ");
596 let segments = [
597 EncodeSegment::new("<ctl>", true),
598 EncodeSegment::new(&user, false),
599 ];
600 let expected = inner.encode_segments(&segments).unwrap();
601 let (cached, events) = collect_token_usage(
602 CachedTokenizer::new(inner, Vec::new(), 4096)
603 .expect("test tokenizer supports prefix caching"),
604 );
605
606 for _ in 0..2 {
607 let actual = cached.encode_segments(&segments).unwrap();
608 assert_eq!(actual.token_ids(), expected.token_ids());
609 }
610 let stats = cached.cache_stats();
611 assert_eq!((stats.entries, stats.hits, stats.misses), (0, 0, 0));
612 assert!(events.lock().unwrap().is_empty());
613 }
614
615 #[test]
616 fn segmented_encoding_caches_long_segments_and_matches_inner() {
617 let inner: Arc<dyn Tokenizer> = Arc::new(SegmentTokenizer);
618 let user = long_text("user content ");
619 let tool = long_text("<ctl> tool output ");
620 let segments = [
623 EncodeSegment::new("<ctl>", true),
624 EncodeSegment::new(&user, false),
625 EncodeSegment::new("<ctl>", true),
626 EncodeSegment::new("short", false),
627 EncodeSegment::new(&tool, false),
628 EncodeSegment::new(&tool, true),
629 EncodeSegment::new("<ctl>", true),
630 ];
631 let expected = inner.encode_segments(&segments).unwrap();
632 let (cached, events) = collect_token_usage(
633 CachedTokenizer::new(inner.clone(), vec!["<ctl>".to_string()], 4096)
634 .expect("test tokenizer supports prefix caching"),
635 );
636
637 let first = cached.encode_segments(&segments).unwrap();
638 let second = cached.encode_segments(&segments).unwrap();
639 assert_eq!(first.token_ids(), expected.token_ids());
640 assert_eq!(second.token_ids(), expected.token_ids());
641
642 let stats = cached.cache_stats();
643 assert_eq!(stats.entries, 3);
644 assert_eq!((stats.hits, stats.misses), (1, 1));
645 let total = expected.token_ids().len();
647 assert_eq!(
648 events.lock().unwrap().as_slice(),
649 &[
650 CacheTokenUsage {
651 cached_tokens: 0,
652 uncached_tokens: total,
653 },
654 CacheTokenUsage {
655 cached_tokens: 6,
656 uncached_tokens: total - 6,
657 },
658 ]
659 );
660 }
661
662 #[test]
663 fn segment_entries_do_not_collide_with_prefix_entries() {
664 let inner: Arc<dyn Tokenizer> = Arc::new(SegmentTokenizer);
665 let cached = CachedTokenizer::new(inner.clone(), vec!["<ctl>".to_string()], 4096)
666 .expect("test tokenizer supports prefix caching");
667 let text = long_text("prefix text ") + "<ctl>";
670 cached.encode(&(text.clone() + " suffix")).unwrap();
671 for allow_special in [false, true] {
672 let segment = [EncodeSegment::new(&text, allow_special)];
673 for _ in 0..2 {
674 assert_eq!(
675 cached.encode_segments(&segment).unwrap().token_ids(),
676 inner.encode_segments(&segment).unwrap().token_ids()
677 );
678 }
679 }
680 }
681
682 #[test]
683 fn token_observer_reports_full_miss_and_partial_hit_with_and_without_extension() {
684 for extend_on_hit in [false, true] {
685 let tok = inner();
686 let hits = Arc::new(AtomicU64::new(0));
687 let misses = Arc::new(AtomicU64::new(0));
688 let hit_counter = hits.clone();
689 let miss_counter = misses.clone();
690 let cached = CachedTokenizer::new(tok, specials(), 64 * 1024)
691 .expect("TinyLlama must support prefix caching")
692 .with_extend(extend_on_hit)
693 .with_observer(
694 Arc::new(move || {
695 hit_counter.fetch_add(1, Ordering::Relaxed);
696 }),
697 Arc::new(move || {
698 miss_counter.fetch_add(1, Ordering::Relaxed);
699 }),
700 );
701 let (cached, events) = collect_token_usage(cached);
702
703 let shared = "<s>system\nYou are helpful.</s><s>user\n";
704 let first = format!("{shared}First question?</s>");
705 let second = format!("{shared}Second different prompt entirely.</s>");
706
707 let first_encoding = cached.encode(&first).unwrap();
708 let second_encoding = cached.encode(&second).unwrap();
709
710 let events = events.lock().unwrap();
711 assert_eq!(events.len(), 2);
712 assert_eq!(
713 events[0],
714 CacheTokenUsage {
715 cached_tokens: 0,
716 uncached_tokens: first_encoding.token_ids().len(),
717 }
718 );
719 assert!(events[1].cached_tokens > 0);
720 assert!(events[1].uncached_tokens > 0);
721 assert_eq!(
722 events[1].cached_tokens + events[1].uncached_tokens,
723 second_encoding.token_ids().len()
724 );
725 assert_eq!(hits.load(Ordering::Relaxed), 1);
726 assert_eq!(misses.load(Ordering::Relaxed), 1);
727 }
728 }
729
730 #[test]
731 fn token_observer_does_not_report_failed_encodes() {
732 let tokenizer: Arc<dyn Tokenizer> = Arc::new(FailingTokenizer);
733 let (cached, events) = collect_token_usage(
734 CachedTokenizer::new(tokenizer, specials(), 4096)
735 .expect("test tokenizer explicitly supports prefix caching"),
736 );
737
738 assert!(cached.encode("<s>this fails</s>").is_err());
739 assert!(events.lock().unwrap().is_empty());
740 }
741
742 #[test]
743 fn two_turn_chat_correctness_and_hit() {
744 let tok = inner();
745 let cached = CachedTokenizer::new(tok.clone(), specials(), 64 * 1024)
746 .expect("TinyLlama must support prefix caching");
747
748 let template = "<s>system\nYou are helpful.</s><s>user\n";
749 let first = format!("{template}First question?</s>");
750 let second = format!("{template}Second different prompt entirely.</s>");
751
752 let _ = cached.encode(&first).unwrap();
754
755 let cached_second = cached.encode(&second).unwrap();
757 let plain_second = tok.encode(&second).unwrap();
758 assert_eq!(
759 cached_second.token_ids(),
760 plain_second.token_ids(),
761 "cached encode must equal plain encode for second turn"
762 );
763
764 let stats = cached.cache_stats();
765 assert!(stats.hits >= 1, "expected L1 hit on second request");
766 }
767
768 #[test]
769 fn decode_passes_through() {
770 let tok = inner();
771 let cached = CachedTokenizer::new(tok.clone(), specials(), 4096)
772 .expect("TinyLlama must support prefix caching");
773 let enc = cached.encode("<s>hello</s>").unwrap();
774 let direct = tok.decode(enc.token_ids(), false).unwrap();
775 let through = cached.decode(enc.token_ids(), false).unwrap();
776 assert_eq!(direct, through);
777 }
778
779 #[test]
780 fn encode_batch_uses_cache() {
781 let tok = inner();
782 let (cached, events) = collect_token_usage(
783 CachedTokenizer::new(tok.clone(), specials(), 64 * 1024)
784 .expect("TinyLlama must support prefix caching"),
785 );
786 let shared = "<s>system\nShared persona.</s><s>user\n";
787 let inputs = [
788 format!("{shared}q1</s>"),
789 format!("{shared}q2</s>"),
790 format!("{shared}q3</s>"),
791 ];
792 let refs: Vec<&str> = inputs.iter().map(String::as_str).collect();
793 let outs = cached.encode_batch(&refs).unwrap();
794 assert_eq!(outs.len(), 3);
795 let events = events.lock().unwrap();
796 assert_eq!(events.len(), outs.len());
797 for (event, output) in events.iter().zip(&outs) {
798 assert_eq!(
799 event.cached_tokens + event.uncached_tokens,
800 output.token_ids().len()
801 );
802 }
803 assert_eq!(events[0].cached_tokens, 0);
804 assert!(events[1..].iter().all(|event| event.cached_tokens > 0));
805 assert!(cached.cache_stats().hits >= 2, "expected hits on q2 and q3");
807 }
808
809 #[test]
810 fn vocab_introspection_forwards_to_inner() {
811 let tok = inner();
812 let cached = CachedTokenizer::new(tok.clone(), specials(), 4096)
813 .expect("TinyLlama must support prefix caching");
814 assert_eq!(cached.vocab_size(), tok.vocab_size());
815 assert_eq!(
816 cached.token_to_id("<s>").unwrap(),
817 tok.token_to_id("<s>").unwrap()
818 );
819 assert_eq!(
820 cached.special_token_ids().unwrap(),
821 tok.special_token_ids().unwrap()
822 );
823 }
824
825 #[test]
826 fn special_token_accounting_matches_cached_encoder_behavior() {
827 let cached = CachedTokenizer::new(inner(), specials(), 4096)
828 .expect("TinyLlama must support prefix caching")
829 .with_options(crate::TokenizerOptions {
830 add_special_tokens: true,
831 });
832 let cached_ids = cached.encode("hello").unwrap();
833 let hf_ids = HuggingFaceTokenizer::from_file(TINYLLAMA_PATH)
834 .expect("load TinyLlama")
835 .with_options(crate::TokenizerOptions {
836 add_special_tokens: true,
837 })
838 .encode("hello")
839 .unwrap();
840
841 assert_eq!(cached.num_special_tokens_added().unwrap(), 0);
842 assert_eq!(hf_ids.token_ids().len(), cached_ids.token_ids().len() + 1);
843 assert_eq!(&hf_ids.token_ids()[1..], cached_ids.token_ids());
844 }
845
846 #[test]
847 fn unoverridden_introspection_methods_use_defaults() {
848 let tokenizer = SegmentTokenizer;
849 assert_eq!(tokenizer.vocab_size(), None);
850 assert!(tokenizer.token_to_id("anything").is_err());
851 assert!(tokenizer.special_token_ids().is_err());
852 assert!(tokenizer.num_special_tokens_added().is_err());
853 }
854}