Skip to main content

dynamo_tokenizers/
fastokens.rs

1// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Fastokens backend using the `fastokens` crate for high-performance BPE encoding.
5//!
6//! This module preserves the existing hybrid behavior: `fastokens` handles encoding and
7//! `HuggingFaceTokenizer` handles decoding. Both are loaded from the same `tokenizer.json`
8//! file.
9//!
10//! [`FastTikTokenTokenizer`] instead loads a bare `tiktoken.model` and decodes its ranks
11//! directly.
12
13use std::collections::HashSet;
14use std::path::Path;
15
16use rayon::prelude::*;
17use rustc_hash::FxHashMap;
18
19use super::{
20    EncodeSegment, Encoding, Error, Result, TokenIdType,
21    hf::HuggingFaceTokenizer,
22    tiktoken,
23    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
24};
25
26fn fast_encode(encoder: &fastokens::Tokenizer, input: &str) -> Result<Encoding> {
27    let ids = encoder
28        .encode(input)
29        .map_err(|e| Error::msg(format!("Fastokens encode error: {e}")))?;
30    Ok(Encoding::Sp(ids))
31}
32
33fn fast_encode_segments(
34    encoder: &fastokens::Tokenizer,
35    segments: &[EncodeSegment<'_>],
36) -> Result<Encoding> {
37    let segments: Vec<fastokens::EncodeSegment<'_>> = segments
38        .iter()
39        .map(|segment| fastokens::EncodeSegment {
40            text: segment.text,
41            allow_special: segment.allow_special,
42        })
43        .collect();
44    let ids = encoder
45        .encode_segments(&segments)
46        .map_err(|e| Error::msg(format!("Fastokens segmented encode error: {e}")))?;
47    Ok(Encoding::Sp(ids))
48}
49
50/// Hybrid tokenizer: fast BPE encoding via `fastokens`, decoding via HuggingFace.
51///
52/// Both backends are loaded from the same `tokenizer.json` file.
53pub struct FastTokenizer {
54    fast_encoder: fastokens::Tokenizer,
55    hf_decoder: HuggingFaceTokenizer,
56}
57
58impl FastTokenizer {
59    pub fn from_file(path: &str) -> Result<Self> {
60        let fast_encoder = fastokens::Tokenizer::from_file(Path::new(path))
61            .map_err(|e| Error::msg(format!("Error loading fastokens tokenizer: {e}")))?;
62        let hf_decoder = HuggingFaceTokenizer::from_file(path)?;
63        Ok(Self {
64            fast_encoder,
65            hf_decoder,
66        })
67    }
68}
69
70impl Encoder for FastTokenizer {
71    fn encode(&self, input: &str) -> Result<Encoding> {
72        fast_encode(&self.fast_encoder, input)
73    }
74
75    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
76        inputs.par_iter().map(|input| self.encode(input)).collect()
77    }
78
79    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
80        fast_encode_segments(&self.fast_encoder, segments)
81    }
82}
83
84impl Decoder for FastTokenizer {
85    fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
86        self.hf_decoder
87            .has_unstable_suffix(token_ids, skip_special_tokens)
88    }
89
90    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
91        self.hf_decoder.decode(token_ids, skip_special_tokens)
92    }
93}
94
95impl Tokenizer for FastTokenizer {
96    fn validate_prefix_cache(&self) -> Result<()> {
97        Ok(())
98    }
99
100    // `fast_encoder` and `hf_decoder` are loaded from the same tokenizer.json,
101    // so the HF side's vocabulary introspection applies to both.
102    fn vocab_size(&self) -> Option<usize> {
103        self.hf_decoder.vocab_size()
104    }
105
106    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
107        self.hf_decoder.token_to_id(token)
108    }
109
110    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
111        self.hf_decoder.special_token_ids()
112    }
113
114    fn num_special_tokens_added(&self) -> Result<usize> {
115        Ok(0)
116    }
117}
118
119/// `fastokens` over a bare `tiktoken.model`, for checkpoints such as Kimi K2/K3 that ship no
120/// `tokenizer.json`. Loads the same ranks, regex, and special tokens as
121/// [`TikTokenTokenizer::from_file_auto`](crate::TikTokenTokenizer::from_file_auto).
122///
123/// Plain-text prefix caching ([`CachedTokenizer`](crate::CachedTokenizer)) is rejected for
124/// this backend pending validation of the full cache matrix. In fastokens 0.3.4, the
125/// known Unicode contraction case is consistent across a special-token boundary, but
126/// still differs from tiktoken-rs. Segmented encodes are unaffected by the cache gate:
127/// every segment is encoded on its own with or without a cache.
128pub struct FastTikTokenTokenizer {
129    inner: fastokens::Tokenizer,
130    /// Decoding joins raw bytes itself because `fastokens`' decoder returns a lossy `String`,
131    /// which cannot tell a vocabulary token ending in `EF BF BD` from a truncated sequence.
132    id_to_bytes: FxHashMap<u32, Vec<u8>>,
133    special_token_ids: HashSet<u32>,
134    special_tokens: Vec<String>,
135}
136
137impl FastTikTokenTokenizer {
138    /// Load a tiktoken model file, reading the BPE pattern from `config.json` and the special
139    /// tokens from `tokenizer_config.json` in the same directory.
140    pub fn from_file_auto(path: &str) -> Result<Self> {
141        let directory = Path::new(path)
142            .parent()
143            .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
144        let pattern = tiktoken::detect_bpe_pattern(directory)?;
145        let encoder = tiktoken::parse_tiktoken_file(path)?;
146        let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
147        let special_tokens = tiktoken::load_special_tokens(directory, num_base_tokens)?;
148
149        let ranks: Vec<(Vec<u8>, u32)> = encoder.into_iter().collect();
150        let mut id_to_bytes: FxHashMap<u32, Vec<u8>> = ranks
151            .iter()
152            .map(|(bytes, rank)| (*rank, bytes.clone()))
153            .collect();
154        id_to_bytes.extend(
155            special_tokens
156                .iter()
157                .map(|(content, &id)| (id, content.as_bytes().to_vec())),
158        );
159        let special_token_ids = special_tokens.values().copied().collect();
160        let special_token_strings = tiktoken::sorted_special_token_strings(&special_tokens);
161
162        let config =
163            fastokens::tiktoken::TiktokenConfig::new(pattern, special_tokens.into_iter().collect());
164        let inner = fastokens::Tokenizer::from_tiktoken_ranks(&ranks, config).map_err(|e| {
165            Error::msg(format!(
166                "Error loading fastokens tiktoken tokenizer from {path}: {e}"
167            ))
168        })?;
169        Ok(Self {
170            inner,
171            id_to_bytes,
172            special_token_ids,
173            special_tokens: special_token_strings,
174        })
175    }
176
177    /// Atomic special-token strings registered with the encoder, sorted; the boundary set
178    /// for [`CachedTokenizer`](crate::CachedTokenizer).
179    pub fn special_tokens(&self) -> &[String] {
180        &self.special_tokens
181    }
182}
183
184impl Encoder for FastTikTokenTokenizer {
185    fn encode(&self, input: &str) -> Result<Encoding> {
186        fast_encode(&self.inner, input)
187    }
188
189    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
190        inputs.par_iter().map(|input| self.encode(input)).collect()
191    }
192
193    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
194        fast_encode_segments(&self.inner, segments)
195    }
196}
197
198impl Decoder for FastTikTokenTokenizer {
199    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
200        let mut bytes = Vec::new();
201        for id in token_ids {
202            if skip_special_tokens && self.special_token_ids.contains(id) {
203                continue;
204            }
205            if let Some(token) = self.id_to_bytes.get(id) {
206                bytes.extend_from_slice(token);
207            }
208        }
209        match String::from_utf8(bytes) {
210            Ok(text) => Ok(DecodeResult::Complete(text)),
211            Err(e) => Ok(DecodeResult::from_decoded(
212                String::from_utf8_lossy(e.as_bytes()).into_owned(),
213            )),
214        }
215    }
216}
217
218impl Tokenizer for FastTikTokenTokenizer {
219    fn validate_prefix_cache(&self) -> Result<()> {
220        Err(Error::msg(
221            "fastokens over tiktoken.model plain-text prefix caching is disabled pending \
222             validation of the full cache matrix",
223        ))
224    }
225
226    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
227        Ok(self.inner.token_to_id(token))
228    }
229
230    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
231        let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
232        ids.sort_unstable();
233        Ok(ids)
234    }
235
236    fn num_special_tokens_added(&self) -> Result<usize> {
237        Ok(0)
238    }
239}
240
241#[cfg(test)]
242mod tests {
243    use super::*;
244    use crate::{HuggingFaceTokenizer, TokenizerOptions};
245
246    // Minimal synthetic BPE tokenizer with no normalizer or post-processor --
247    // compatible with fastokens. Vocab covers: H,T,a,d,e,h,i,l,o,r,s,t,w + punctuation.
248    const TOKENIZER_PATH: &str = concat!(
249        env!("CARGO_MANIFEST_DIR"),
250        "/tests/data/minimal-bpe/tokenizer.json"
251    );
252    const SEGMENTED_TOKENIZER_PATH: &str = concat!(
253        env!("CARGO_MANIFEST_DIR"),
254        "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
255    );
256
257    #[test]
258    fn byte_fallback_stream_matches_full_decode() {
259        let tokenizer: crate::Tokenizer =
260            std::sync::Arc::new(FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap()).into();
261        for (pieces, skip) in [
262            (vec!["<0x61>", "<0xF5>"], false),
263            (vec!["<0x61>", "</s>", "<0xF5>"], false),
264            (vec!["<0x61>", "</s>", "<0xF5>"], true),
265        ] {
266            let ids: Vec<_> = pieces
267                .iter()
268                .map(|piece| tokenizer.token_to_id(piece).unwrap().unwrap())
269                .collect();
270            let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
271            let mut stream = tokenizer.decode_stream(&[], skip);
272            let mut actual = String::new();
273            for id in ids {
274                actual.push_str(&stream.step(id).unwrap().unwrap_or_default());
275            }
276            actual.push_str(&stream.finish().unwrap().unwrap_or_default());
277            assert_eq!(actual, expected, "{pieces:?}, skip={skip}");
278        }
279    }
280
281    #[test]
282    fn test_fast_encode_decode_roundtrip() {
283        let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
284        // Encode then decode: verifies both paths execute without error.
285        // With a null decoder, HF inserts spaces between tokens so exact equality
286        // is not expected here -- we just verify the operations succeed and produce
287        // non-empty results.
288        let text = "Hello, world!";
289        let encoding = tokenizer.encode(text).unwrap();
290        assert!(!encoding.token_ids().is_empty());
291        let decoded: String = tokenizer.decode(encoding.token_ids(), true).unwrap().into();
292        assert!(!decoded.is_empty());
293        // The decoded text should contain the same non-space characters
294        let enc_chars: String = text.chars().filter(|c| !c.is_whitespace()).collect();
295        let dec_chars: String = decoded.chars().filter(|c| !c.is_whitespace()).collect();
296        assert_eq!(
297            enc_chars, dec_chars,
298            "non-space characters must be preserved"
299        );
300    }
301
302    #[test]
303    fn test_fast_matches_hf_encoding() {
304        let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
305        let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
306
307        for text in &["Hello, world!", "Hello", " world", "He llo"] {
308            let fast_ids = fast.encode(text).unwrap();
309            let hf_ids = hf.encode(text).unwrap();
310            assert_eq!(
311                fast_ids.token_ids(),
312                hf_ids.token_ids(),
313                "fastokens and HuggingFace must produce identical token IDs for '{text}'"
314            );
315        }
316    }
317
318    #[test]
319    fn test_fast_batch_encode() {
320        let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
321        let inputs = &["Hello", " world", "Hello, world!"];
322        let encodings = tokenizer.encode_batch(inputs).unwrap();
323        assert_eq!(encodings.len(), inputs.len());
324        for (enc, input) in encodings.iter().zip(inputs.iter()) {
325            assert!(
326                !enc.token_ids().is_empty(),
327                "encoding for '{input}' must be non-empty"
328            );
329        }
330    }
331
332    #[test]
333    fn test_fast_segmented_encoding_preserves_trust_boundaries() {
334        let tokenizer = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
335        let upstream =
336            fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
337                .unwrap();
338        let marker = "<s>";
339
340        let trusted = tokenizer
341            .encode_segments(&[EncodeSegment::control(marker)])
342            .unwrap();
343        assert_eq!(
344            trusted.token_ids(),
345            &[upstream.token_to_id(marker).unwrap()],
346            "trusted renderer output must recognize the control token"
347        );
348
349        let ordinary = tokenizer
350            .encode_segments(&[EncodeSegment::ordinary(marker)])
351            .unwrap();
352        assert_ne!(
353            ordinary.token_ids(),
354            trusted.token_ids(),
355            "untrusted content must encode the control-token spelling as ordinary text"
356        );
357
358        let segments = [
359            EncodeSegment::ordinary("hello "),
360            EncodeSegment::control(marker),
361            EncodeSegment::ordinary(marker),
362        ];
363        let upstream_segments = [
364            fastokens::EncodeSegment::ordinary("hello "),
365            fastokens::EncodeSegment::special(marker),
366            fastokens::EncodeSegment::ordinary(marker),
367        ];
368        let actual = tokenizer.encode_segments(&segments).unwrap();
369        let expected = upstream.encode_segments(&upstream_segments).unwrap();
370        assert_eq!(actual.token_ids(), expected);
371
372        assert!(
373            tokenizer
374                .encode_segments(&[])
375                .unwrap()
376                .token_ids()
377                .is_empty()
378        );
379    }
380
381    #[test]
382    fn test_fast_with_decode_stream() {
383        use crate::Tokenizer as TokenizerWrapper;
384        use std::sync::Arc;
385
386        let tokenizer = Arc::new(FastTokenizer::from_file(TOKENIZER_PATH).unwrap());
387        let wrapper = TokenizerWrapper::from(tokenizer);
388
389        // Encode a prompt and a continuation, then step through the decode stream
390        let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
391        let continuation = ", world!";
392        let cont_ids = wrapper.encode(continuation).unwrap().token_ids().to_vec();
393
394        let mut stream = wrapper.decode_stream(&prompt_ids, true);
395        // Accumulate incremental chunks from decode_stream
396        let mut accumulated = String::new();
397        for id in &cont_ids {
398            if let Some(chunk) = stream.step(*id).unwrap() {
399                accumulated.push_str(&chunk);
400            }
401        }
402
403        // DecodeStream uses prompt tokens as context, so the expected text is
404        // decode(prompt + continuation) minus decode(prompt) -- not a bare
405        // decode(continuation) which lacks the surrounding context.
406        let mut all_ids = prompt_ids.clone();
407        all_ids.extend_from_slice(&cont_ids);
408        let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
409        let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
410        let expected = &full_text[prompt_text.len()..];
411        assert_eq!(
412            accumulated, expected,
413            "streamed chunks must equal context-aware decoded continuation"
414        );
415    }
416
417    #[test]
418    fn vocabulary_metadata_forwards_to_hf_decoder() {
419        let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
420        let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
421        assert_eq!(fast.vocab_size(), hf.vocab_size());
422        assert_eq!(
423            fast.token_to_id("Hello").unwrap(),
424            hf.token_to_id("Hello").unwrap()
425        );
426        assert_eq!(
427            fast.special_token_ids().unwrap(),
428            hf.special_token_ids().unwrap()
429        );
430    }
431
432    #[test]
433    fn special_token_accounting_matches_fast_encoder() {
434        let fast = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
435        let upstream =
436            fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
437                .unwrap();
438        let hf = HuggingFaceTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
439
440        assert_eq!(hf.num_special_tokens_added().unwrap(), 1);
441        assert_eq!(fast.num_special_tokens_added().unwrap(), 0);
442        let hf_with_special_tokens = hf.with_options(TokenizerOptions {
443            add_special_tokens: true,
444        });
445
446        for text in ["hello", "hello there"] {
447            let fast_ids = fast.encode(text).unwrap();
448            assert_eq!(
449                fast_ids.token_ids(),
450                upstream.encode(text).unwrap(),
451                "FastTokenizer must match the encoder that omits the HF post-processor"
452            );
453            assert_eq!(
454                hf_with_special_tokens
455                    .encode(text)
456                    .unwrap()
457                    .token_ids()
458                    .len(),
459                fast_ids.token_ids().len() + 1,
460                "the HF post-processor must add the BOS token FastTokenizer omits"
461            );
462        }
463    }
464}
465
466#[cfg(test)]
467mod tiktoken_parity_tests {
468    use super::*;
469    use crate::TikTokenTokenizer;
470
471    const TIKTOKEN_PATH: &str = concat!(
472        env!("CARGO_MANIFEST_DIR"),
473        "/tests/data/sample-models/mock-tiktoken-bpe/tiktoken.model"
474    );
475    /// All 256 byte tokens plus the merges ` I` and ` I'`, with the Kimi pattern: the
476    /// smallest vocabulary that exposes fastokens' Unicode contraction mismatch with tiktoken-rs.
477    const CONTRACTION_PATH: &str = concat!(
478        env!("CARGO_MANIFEST_DIR"),
479        "/tests/data/sample-models/mock-tiktoken-contraction/tiktoken.model"
480    );
481
482    fn pair() -> (TikTokenTokenizer, FastTikTokenTokenizer) {
483        let reference = TikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
484        let fast = FastTikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
485        (reference, fast)
486    }
487
488    fn corpus() -> Vec<String> {
489        vec![
490            String::new(),
491            "hello world".into(),
492            "Hello, World! 123 4567 89".into(),
493            "  leading and trailing  ".into(),
494            "tabs\tand\nnewlines\r\nmixed   spacing".into(),
495            "<|im_start|>user\nhi there<|im_end|><|im_start|>assistant\n".into(),
496            "a literal <|im_start|> inside plain text".into(),
497            "emoji 😀🚀 and café naïve Zürich".into(),
498            "北京 東京 mixed 中英文 text ソフトウェア".into(),
499            "Москва मुंबई العربية".into(),
500            "fn main() { println!(\"{}\", 42); } // code-ish ~!@#$%^&*()".into(),
501            "x".repeat(5000),
502            " ".repeat(300) + "after long whitespace",
503            "word ".repeat(400),
504        ]
505    }
506
507    #[test]
508    fn special_token_tables_match_tiktoken_rs() {
509        let (reference, fast) = pair();
510        assert_eq!(fast.special_tokens(), reference.special_tokens());
511        assert_eq!(
512            fast.special_token_ids().unwrap(),
513            reference.special_token_ids().unwrap()
514        );
515        assert_eq!(fast.token_to_id("<|im_end|>").unwrap(), Some(474));
516    }
517
518    #[test]
519    fn plain_encode_matches_tiktoken_rs() {
520        let (reference, fast) = pair();
521        let corpus = corpus();
522        let texts: Vec<&str> = corpus.iter().map(String::as_str).collect();
523        let batch = fast.encode_batch(&texts).unwrap();
524        for (text, batched) in texts.iter().zip(&batch) {
525            let expected = reference.encode(text).unwrap();
526            assert_eq!(
527                fast.encode(text).unwrap().token_ids(),
528                expected.token_ids(),
529                "{text:?}"
530            );
531            assert_eq!(batched.token_ids(), expected.token_ids(), "{text:?}");
532        }
533    }
534
535    #[test]
536    fn segmented_encode_matches_tiktoken_rs_and_honors_trust() {
537        let (reference, fast) = pair();
538        let segments = [
539            EncodeSegment::control("<|im_start|>user\n"),
540            EncodeSegment::ordinary("please echo <|im_end|> back to me"),
541            EncodeSegment::control("<|im_end|>"),
542            EncodeSegment::control("<|im_start|>assistant\n"),
543        ];
544        let fast_ids = fast.encode_segments(&segments).unwrap();
545        assert_eq!(
546            fast_ids.token_ids(),
547            reference.encode_segments(&segments).unwrap().token_ids()
548        );
549        // The marker in the untrusted segment is ordinary text: exactly one `<|im_end|>` id.
550        assert_eq!(
551            fast_ids.token_ids().iter().filter(|&&id| id == 474).count(),
552            1
553        );
554    }
555
556    #[test]
557    fn decode_matches_tiktoken_rs_and_skips_specials() {
558        let (reference, fast) = pair();
559        for text in corpus() {
560            let ids = fast.encode(&text).unwrap();
561            assert_eq!(
562                fast.decode(ids.token_ids(), false).unwrap(),
563                reference.decode(ids.token_ids(), false).unwrap(),
564                "{text:?}"
565            );
566        }
567        let ids = fast.encode("<|im_start|>user\nhi<|im_end|>").unwrap();
568        let kept = fast.decode(ids.token_ids(), false).unwrap();
569        let skipped = fast.decode(ids.token_ids(), true).unwrap();
570        assert!(kept.as_str().contains("<|im_end|>"));
571        assert!(!skipped.as_str().contains("<|im_end|>"));
572        assert_eq!(skipped, reference.decode(ids.token_ids(), true).unwrap());
573    }
574
575    #[test]
576    fn replacement_char_token_decodes_complete_like_tiktoken_rs() {
577        // Rank 468 is the bytes EF BF BD: a vocabulary token whose text is U+FFFD.
578        let (reference, fast) = pair();
579        let decoded = fast.decode(&[468], false).unwrap();
580        assert!(decoded.is_complete(), "{decoded:?}");
581        assert_eq!(decoded, reference.decode(&[468], false).unwrap());
582
583        // A truncated multi-byte sequence stays `Partial`.
584        let emoji = fast.encode("😀").unwrap();
585        let cut = &emoji.token_ids()[..emoji.token_ids().len() - 1];
586        let fast_cut = fast.decode(cut, false).unwrap();
587        assert!(fast_cut.is_partial(), "{fast_cut:?}");
588        assert_eq!(fast_cut, reference.decode(cut, false).unwrap());
589
590        // Unknown ids are skipped.
591        assert_eq!(fast.decode(&[468, 9_999_999], false).unwrap(), decoded);
592    }
593
594    #[test]
595    fn plain_text_prefix_cache_is_rejected() {
596        let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
597        let specials = fast.special_tokens().to_vec();
598        let error = crate::CachedTokenizer::new(std::sync::Arc::new(fast), specials, 1 << 20)
599            .err()
600            .expect("fastokens over tiktoken.model must not be prefix-cached")
601            .to_string();
602        assert!(error.contains("fastokens"), "{error}");
603    }
604
605    /// Fastokens 0.3.4 is consistent across this special-token boundary, but still
606    /// differs from tiktoken-rs on the Unicode case folding of `(?i:'s)` (`'ſ`).
607    /// This single case does not establish the full plain-text prefix-cache contract.
608    #[test]
609    fn unicode_contraction_is_consistent_across_special_token_boundary() {
610        let reference = TikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
611        let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
612        let special = "<|end_of_msg|>";
613        let suffix = " I'\u{17f}";
614        let full = format!("{special}{suffix}");
615
616        let ref_full = reference.encode(&full).unwrap().token_ids().to_vec();
617        let ref_suffix = reference.encode(suffix).unwrap().token_ids().to_vec();
618        assert_eq!(
619            ref_full[1..],
620            ref_suffix[..],
621            "tiktoken-rs must be self-consistent"
622        );
623
624        let fast_full = fast.encode(&full).unwrap().token_ids().to_vec();
625        let fast_suffix = fast.encode(suffix).unwrap().token_ids().to_vec();
626        assert_ne!(
627            fast_full, ref_full,
628            "revisit the known Unicode contraction mismatch"
629        );
630        assert_eq!(
631            fast_full[1..],
632            fast_suffix[..],
633            "splitting at this special-token boundary must preserve fastokens ids"
634        );
635    }
636}