Skip to main content

dynamo_tokenizers/
fastokens.rs

1// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Fastokens backend using the `fastokens` crate for high-performance BPE encoding.
5//!
6//! This module preserves the existing hybrid behavior: `fastokens` handles encoding and
7//! `HuggingFaceTokenizer` handles decoding. Both are loaded from the same `tokenizer.json`
8//! file.
9//!
10//! [`FastTikTokenTokenizer`] instead loads a bare `tiktoken.model` and decodes its ranks
11//! directly.
12
13use std::collections::HashSet;
14use std::path::Path;
15
16use rayon::prelude::*;
17use rustc_hash::FxHashMap;
18
19use super::{
20    EncodeSegment, Encoding, Error, Result, TokenIdType,
21    hf::HuggingFaceTokenizer,
22    tiktoken,
23    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
24};
25
26fn fast_encode(encoder: &fastokens::Tokenizer, input: &str) -> Result<Encoding> {
27    let ids = encoder
28        .encode(input)
29        .map_err(|e| Error::msg(format!("Fastokens encode error: {e}")))?;
30    Ok(Encoding::Sp(ids))
31}
32
33fn fast_encode_segments(
34    encoder: &fastokens::Tokenizer,
35    segments: &[EncodeSegment<'_>],
36) -> Result<Encoding> {
37    let segments: Vec<fastokens::EncodeSegment<'_>> = segments
38        .iter()
39        .map(|segment| fastokens::EncodeSegment {
40            text: segment.text,
41            allow_special: segment.allow_special,
42        })
43        .collect();
44    let ids = encoder
45        .encode_segments(&segments)
46        .map_err(|e| Error::msg(format!("Fastokens segmented encode error: {e}")))?;
47    Ok(Encoding::Sp(ids))
48}
49
50/// Hybrid tokenizer: fast BPE encoding via `fastokens`, decoding via HuggingFace.
51///
52/// Both backends are loaded from the same `tokenizer.json` file.
53pub struct FastTokenizer {
54    fast_encoder: fastokens::Tokenizer,
55    hf_decoder: HuggingFaceTokenizer,
56}
57
58impl FastTokenizer {
59    pub fn from_file(path: &str) -> Result<Self> {
60        let fast_encoder = fastokens::Tokenizer::from_file(Path::new(path))
61            .map_err(|e| Error::msg(format!("Error loading fastokens tokenizer: {e}")))?;
62        let hf_decoder = HuggingFaceTokenizer::from_file(path)?;
63        Ok(Self {
64            fast_encoder,
65            hf_decoder,
66        })
67    }
68}
69
70impl Encoder for FastTokenizer {
71    fn encode(&self, input: &str) -> Result<Encoding> {
72        fast_encode(&self.fast_encoder, input)
73    }
74
75    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
76        inputs.par_iter().map(|input| self.encode(input)).collect()
77    }
78
79    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
80        fast_encode_segments(&self.fast_encoder, segments)
81    }
82}
83
84impl Decoder for FastTokenizer {
85    fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
86        self.hf_decoder
87            .has_unstable_suffix(token_ids, skip_special_tokens)
88    }
89
90    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
91        self.hf_decoder.decode(token_ids, skip_special_tokens)
92    }
93}
94
95impl Tokenizer for FastTokenizer {
96    fn validate_prefix_cache(&self) -> Result<()> {
97        Ok(())
98    }
99
100    // `fast_encoder` and `hf_decoder` are loaded from the same tokenizer.json,
101    // so the HF side's vocabulary introspection applies to both.
102    fn vocab_size(&self) -> Option<usize> {
103        self.hf_decoder.vocab_size()
104    }
105
106    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
107        self.hf_decoder.token_to_id(token)
108    }
109
110    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
111        self.hf_decoder.special_token_ids()
112    }
113
114    fn num_special_tokens_added(&self) -> Result<usize> {
115        Ok(0)
116    }
117}
118
119/// `fastokens` over a bare `tiktoken.model`, for checkpoints such as Kimi K2/K3 that ship no
120/// `tokenizer.json`. Loads the same ranks, regex, and special tokens as
121/// [`TikTokenTokenizer::from_file_auto`](crate::TikTokenTokenizer::from_file_auto).
122///
123/// Plain-text prefix caching ([`CachedTokenizer`](crate::CachedTokenizer)) is rejected for
124/// this backend pending validation of the full cache matrix. In fastokens 0.3.4, the
125/// known Unicode contraction case is consistent across a special-token boundary, but
126/// still differs from tiktoken-rs. Segmented encodes are unaffected by the cache gate:
127/// every segment is encoded on its own with or without a cache.
128pub struct FastTikTokenTokenizer {
129    inner: fastokens::Tokenizer,
130    /// Decoding joins raw bytes itself because `fastokens`' decoder returns a lossy `String`,
131    /// which cannot tell a vocabulary token ending in `EF BF BD` from a truncated sequence.
132    id_to_bytes: FxHashMap<u32, Vec<u8>>,
133    special_token_ids: HashSet<u32>,
134    special_tokens: Vec<String>,
135}
136
137impl FastTikTokenTokenizer {
138    /// Load a tiktoken model file, reading the BPE pattern from `config.json` and the special
139    /// tokens from `tokenizer_config.json` in the same directory.
140    pub fn from_file_auto(path: &str) -> Result<Self> {
141        let directory = Path::new(path)
142            .parent()
143            .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
144        let pattern = tiktoken::detect_bpe_pattern(directory)?;
145        let encoder = tiktoken::parse_tiktoken_file(path)?;
146        let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
147        let tiktoken::AddedTokens { tokens, skip_ids } =
148            tiktoken::load_added_tokens(directory, num_base_tokens)?;
149
150        let ranks: Vec<(Vec<u8>, u32)> = encoder.into_iter().collect();
151        let mut id_to_bytes: FxHashMap<u32, Vec<u8>> = ranks
152            .iter()
153            .map(|(bytes, rank)| (*rank, bytes.clone()))
154            .collect();
155        id_to_bytes.extend(
156            tokens
157                .iter()
158                .map(|(content, &id)| (id, content.as_bytes().to_vec())),
159        );
160        let special_token_strings = tiktoken::sorted_special_token_strings(&tokens);
161
162        let config =
163            fastokens::tiktoken::TiktokenConfig::new(pattern, tokens.into_iter().collect());
164        let inner = fastokens::Tokenizer::from_tiktoken_ranks(&ranks, config).map_err(|e| {
165            Error::msg(format!(
166                "Error loading fastokens tiktoken tokenizer from {path}: {e}"
167            ))
168        })?;
169        Ok(Self {
170            inner,
171            id_to_bytes,
172            special_token_ids: skip_ids,
173            special_tokens: special_token_strings,
174        })
175    }
176
177    /// Atomic special-token strings registered with the encoder, sorted; the boundary set
178    /// for [`CachedTokenizer`](crate::CachedTokenizer). Includes added tokens marked
179    /// `"special": false`, unlike `special_token_ids`.
180    pub fn special_tokens(&self) -> &[String] {
181        &self.special_tokens
182    }
183}
184
185impl Encoder for FastTikTokenTokenizer {
186    fn encode(&self, input: &str) -> Result<Encoding> {
187        fast_encode(&self.inner, input)
188    }
189
190    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
191        inputs.par_iter().map(|input| self.encode(input)).collect()
192    }
193
194    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
195        fast_encode_segments(&self.inner, segments)
196    }
197}
198
199impl Decoder for FastTikTokenTokenizer {
200    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
201        let mut bytes = Vec::new();
202        for id in token_ids {
203            if skip_special_tokens && self.special_token_ids.contains(id) {
204                continue;
205            }
206            if let Some(token) = self.id_to_bytes.get(id) {
207                bytes.extend_from_slice(token);
208            }
209        }
210        match String::from_utf8(bytes) {
211            Ok(text) => Ok(DecodeResult::Complete(text)),
212            Err(e) => Ok(DecodeResult::from_decoded(
213                String::from_utf8_lossy(e.as_bytes()).into_owned(),
214            )),
215        }
216    }
217}
218
219impl Tokenizer for FastTikTokenTokenizer {
220    fn validate_prefix_cache(&self) -> Result<()> {
221        Err(Error::msg(
222            "fastokens over tiktoken.model plain-text prefix caching is disabled pending \
223             validation of the full cache matrix",
224        ))
225    }
226
227    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
228        Ok(self.inner.token_to_id(token))
229    }
230
231    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
232        let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
233        ids.sort_unstable();
234        Ok(ids)
235    }
236
237    fn num_special_tokens_added(&self) -> Result<usize> {
238        Ok(0)
239    }
240}
241
242#[cfg(test)]
243mod tests {
244    use super::*;
245    use crate::{HuggingFaceTokenizer, TokenizerOptions};
246
247    // Minimal synthetic BPE tokenizer with no normalizer or post-processor --
248    // compatible with fastokens. Vocab covers: H,T,a,d,e,h,i,l,o,r,s,t,w + punctuation.
249    const TOKENIZER_PATH: &str = concat!(
250        env!("CARGO_MANIFEST_DIR"),
251        "/tests/data/minimal-bpe/tokenizer.json"
252    );
253    const SEGMENTED_TOKENIZER_PATH: &str = concat!(
254        env!("CARGO_MANIFEST_DIR"),
255        "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
256    );
257
258    #[test]
259    fn byte_fallback_stream_matches_full_decode() {
260        let tokenizer: crate::Tokenizer =
261            std::sync::Arc::new(FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap()).into();
262        for (pieces, skip) in [
263            (vec!["<0x61>", "<0xF5>"], false),
264            (vec!["<0x61>", "</s>", "<0xF5>"], false),
265            (vec!["<0x61>", "</s>", "<0xF5>"], true),
266        ] {
267            let ids: Vec<_> = pieces
268                .iter()
269                .map(|piece| tokenizer.token_to_id(piece).unwrap().unwrap())
270                .collect();
271            let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
272            let mut stream = tokenizer.decode_stream(&[], skip);
273            let mut actual = String::new();
274            for id in ids {
275                actual.push_str(&stream.step(id).unwrap().unwrap_or_default());
276            }
277            actual.push_str(&stream.finish().unwrap().unwrap_or_default());
278            assert_eq!(actual, expected, "{pieces:?}, skip={skip}");
279        }
280    }
281
282    #[test]
283    fn test_fast_encode_decode_roundtrip() {
284        let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
285        // Encode then decode: verifies both paths execute without error.
286        // With a null decoder, HF inserts spaces between tokens so exact equality
287        // is not expected here -- we just verify the operations succeed and produce
288        // non-empty results.
289        let text = "Hello, world!";
290        let encoding = tokenizer.encode(text).unwrap();
291        assert!(!encoding.token_ids().is_empty());
292        let decoded: String = tokenizer.decode(encoding.token_ids(), true).unwrap().into();
293        assert!(!decoded.is_empty());
294        // The decoded text should contain the same non-space characters
295        let enc_chars: String = text.chars().filter(|c| !c.is_whitespace()).collect();
296        let dec_chars: String = decoded.chars().filter(|c| !c.is_whitespace()).collect();
297        assert_eq!(
298            enc_chars, dec_chars,
299            "non-space characters must be preserved"
300        );
301    }
302
303    #[test]
304    fn test_fast_matches_hf_encoding() {
305        let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
306        let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
307
308        for text in &["Hello, world!", "Hello", " world", "He llo"] {
309            let fast_ids = fast.encode(text).unwrap();
310            let hf_ids = hf.encode(text).unwrap();
311            assert_eq!(
312                fast_ids.token_ids(),
313                hf_ids.token_ids(),
314                "fastokens and HuggingFace must produce identical token IDs for '{text}'"
315            );
316        }
317    }
318
319    #[test]
320    fn test_fast_batch_encode() {
321        let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
322        let inputs = &["Hello", " world", "Hello, world!"];
323        let encodings = tokenizer.encode_batch(inputs).unwrap();
324        assert_eq!(encodings.len(), inputs.len());
325        for (enc, input) in encodings.iter().zip(inputs.iter()) {
326            assert!(
327                !enc.token_ids().is_empty(),
328                "encoding for '{input}' must be non-empty"
329            );
330        }
331    }
332
333    #[test]
334    fn test_fast_segmented_encoding_preserves_trust_boundaries() {
335        let tokenizer = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
336        let upstream =
337            fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
338                .unwrap();
339        let marker = "<s>";
340
341        let trusted = tokenizer
342            .encode_segments(&[EncodeSegment::control(marker)])
343            .unwrap();
344        assert_eq!(
345            trusted.token_ids(),
346            &[upstream.token_to_id(marker).unwrap()],
347            "trusted renderer output must recognize the control token"
348        );
349
350        let ordinary = tokenizer
351            .encode_segments(&[EncodeSegment::ordinary(marker)])
352            .unwrap();
353        assert_ne!(
354            ordinary.token_ids(),
355            trusted.token_ids(),
356            "untrusted content must encode the control-token spelling as ordinary text"
357        );
358
359        let segments = [
360            EncodeSegment::ordinary("hello "),
361            EncodeSegment::control(marker),
362            EncodeSegment::ordinary(marker),
363        ];
364        let upstream_segments = [
365            fastokens::EncodeSegment::ordinary("hello "),
366            fastokens::EncodeSegment::special(marker),
367            fastokens::EncodeSegment::ordinary(marker),
368        ];
369        let actual = tokenizer.encode_segments(&segments).unwrap();
370        let expected = upstream.encode_segments(&upstream_segments).unwrap();
371        assert_eq!(actual.token_ids(), expected);
372
373        assert!(
374            tokenizer
375                .encode_segments(&[])
376                .unwrap()
377                .token_ids()
378                .is_empty()
379        );
380    }
381
382    #[test]
383    fn test_fast_with_decode_stream() {
384        use crate::Tokenizer as TokenizerWrapper;
385        use std::sync::Arc;
386
387        let tokenizer = Arc::new(FastTokenizer::from_file(TOKENIZER_PATH).unwrap());
388        let wrapper = TokenizerWrapper::from(tokenizer);
389
390        // Encode a prompt and a continuation, then step through the decode stream
391        let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
392        let continuation = ", world!";
393        let cont_ids = wrapper.encode(continuation).unwrap().token_ids().to_vec();
394
395        let mut stream = wrapper.decode_stream(&prompt_ids, true);
396        // Accumulate incremental chunks from decode_stream
397        let mut accumulated = String::new();
398        for id in &cont_ids {
399            if let Some(chunk) = stream.step(*id).unwrap() {
400                accumulated.push_str(&chunk);
401            }
402        }
403
404        // DecodeStream uses prompt tokens as context, so the expected text is
405        // decode(prompt + continuation) minus decode(prompt) -- not a bare
406        // decode(continuation) which lacks the surrounding context.
407        let mut all_ids = prompt_ids.clone();
408        all_ids.extend_from_slice(&cont_ids);
409        let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
410        let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
411        let expected = &full_text[prompt_text.len()..];
412        assert_eq!(
413            accumulated, expected,
414            "streamed chunks must equal context-aware decoded continuation"
415        );
416    }
417
418    #[test]
419    fn vocabulary_metadata_forwards_to_hf_decoder() {
420        let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
421        let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
422        assert_eq!(fast.vocab_size(), hf.vocab_size());
423        assert_eq!(
424            fast.token_to_id("Hello").unwrap(),
425            hf.token_to_id("Hello").unwrap()
426        );
427        assert_eq!(
428            fast.special_token_ids().unwrap(),
429            hf.special_token_ids().unwrap()
430        );
431    }
432
433    #[test]
434    fn special_token_accounting_matches_fast_encoder() {
435        let fast = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
436        let upstream =
437            fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
438                .unwrap();
439        let hf = HuggingFaceTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
440
441        assert_eq!(hf.num_special_tokens_added().unwrap(), 1);
442        assert_eq!(fast.num_special_tokens_added().unwrap(), 0);
443        let hf_with_special_tokens = hf.with_options(TokenizerOptions {
444            add_special_tokens: true,
445        });
446
447        for text in ["hello", "hello there"] {
448            let fast_ids = fast.encode(text).unwrap();
449            assert_eq!(
450                fast_ids.token_ids(),
451                upstream.encode(text).unwrap(),
452                "FastTokenizer must match the encoder that omits the HF post-processor"
453            );
454            assert_eq!(
455                hf_with_special_tokens
456                    .encode(text)
457                    .unwrap()
458                    .token_ids()
459                    .len(),
460                fast_ids.token_ids().len() + 1,
461                "the HF post-processor must add the BOS token FastTokenizer omits"
462            );
463        }
464    }
465}
466
467#[cfg(test)]
468mod tiktoken_parity_tests {
469    use super::*;
470    use crate::TikTokenTokenizer;
471
472    const TIKTOKEN_PATH: &str = concat!(
473        env!("CARGO_MANIFEST_DIR"),
474        "/tests/data/sample-models/mock-tiktoken-bpe/tiktoken.model"
475    );
476    /// All 256 byte tokens plus the merges ` I` and ` I'`, with the Kimi pattern: the
477    /// smallest vocabulary that exposes fastokens' Unicode contraction mismatch with tiktoken-rs.
478    const CONTRACTION_PATH: &str = concat!(
479        env!("CARGO_MANIFEST_DIR"),
480        "/tests/data/sample-models/mock-tiktoken-contraction/tiktoken.model"
481    );
482
483    fn pair() -> (TikTokenTokenizer, FastTikTokenTokenizer) {
484        let reference = TikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
485        let fast = FastTikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
486        (reference, fast)
487    }
488
489    fn corpus() -> Vec<String> {
490        vec![
491            String::new(),
492            "hello world".into(),
493            "Hello, World! 123 4567 89".into(),
494            "  leading and trailing  ".into(),
495            "tabs\tand\nnewlines\r\nmixed   spacing".into(),
496            "<|im_start|>user\nhi there<|im_end|><|im_start|>assistant\n".into(),
497            "a literal <|im_start|> inside plain text".into(),
498            "emoji 😀🚀 and café naïve Zürich".into(),
499            "北京 東京 mixed 中英文 text ソフトウェア".into(),
500            "Москва मुंबई العربية".into(),
501            "fn main() { println!(\"{}\", 42); } // code-ish ~!@#$%^&*()".into(),
502            "x".repeat(5000),
503            " ".repeat(300) + "after long whitespace",
504            "word ".repeat(400),
505        ]
506    }
507
508    #[test]
509    fn special_token_tables_match_tiktoken_rs() {
510        let (reference, fast) = pair();
511        assert_eq!(fast.special_tokens(), reference.special_tokens());
512        assert_eq!(
513            fast.special_token_ids().unwrap(),
514            reference.special_token_ids().unwrap()
515        );
516        assert_eq!(fast.token_to_id("<|im_end|>").unwrap(), Some(474));
517    }
518
519    #[test]
520    fn non_special_added_tokens_survive_skip_special_tokens() {
521        let source = Path::new(TIKTOKEN_PATH).parent().unwrap();
522        let dir = tempfile::tempdir().unwrap();
523        for file in ["tiktoken.model", "config.json"] {
524            std::fs::copy(source.join(file), dir.path().join(file)).unwrap();
525        }
526        let mut config: serde_json::Value = serde_json::from_str(
527            &std::fs::read_to_string(source.join("tokenizer_config.json")).unwrap(),
528        )
529        .unwrap();
530        config["added_tokens_decoder"]["475"] =
531            serde_json::json!({"content": "<|open|>", "special": false});
532        std::fs::write(dir.path().join("tokenizer_config.json"), config.to_string()).unwrap();
533        let path = dir.path().join("tiktoken.model");
534        let path = path.to_str().unwrap();
535        let reference = TikTokenTokenizer::from_file_auto(path).unwrap();
536        let fast = FastTikTokenTokenizer::from_file_auto(path).unwrap();
537
538        let special_ids = fast.special_token_ids().unwrap();
539        assert_eq!(special_ids, reference.special_token_ids().unwrap());
540        assert!(special_ids.contains(&474));
541        assert!(!special_ids.contains(&475));
542
543        let ids = fast.encode("<|open|>hello<|im_end|>").unwrap();
544        assert_eq!(
545            ids.token_ids(),
546            reference
547                .encode("<|open|>hello<|im_end|>")
548                .unwrap()
549                .token_ids()
550        );
551        assert_eq!(ids.token_ids().first(), Some(&475));
552        let fast_text: String = fast.decode(ids.token_ids(), true).unwrap().into();
553        let reference_text: String = reference.decode(ids.token_ids(), true).unwrap().into();
554        assert_eq!(fast_text, "<|open|>hello");
555        assert_eq!(reference_text, "<|open|>hello");
556    }
557
558    #[test]
559    fn plain_encode_matches_tiktoken_rs() {
560        let (reference, fast) = pair();
561        let corpus = corpus();
562        let texts: Vec<&str> = corpus.iter().map(String::as_str).collect();
563        let batch = fast.encode_batch(&texts).unwrap();
564        for (text, batched) in texts.iter().zip(&batch) {
565            let expected = reference.encode(text).unwrap();
566            assert_eq!(
567                fast.encode(text).unwrap().token_ids(),
568                expected.token_ids(),
569                "{text:?}"
570            );
571            assert_eq!(batched.token_ids(), expected.token_ids(), "{text:?}");
572        }
573    }
574
575    #[test]
576    fn segmented_encode_matches_tiktoken_rs_and_honors_trust() {
577        let (reference, fast) = pair();
578        let segments = [
579            EncodeSegment::control("<|im_start|>user\n"),
580            EncodeSegment::ordinary("please echo <|im_end|> back to me"),
581            EncodeSegment::control("<|im_end|>"),
582            EncodeSegment::control("<|im_start|>assistant\n"),
583        ];
584        let fast_ids = fast.encode_segments(&segments).unwrap();
585        assert_eq!(
586            fast_ids.token_ids(),
587            reference.encode_segments(&segments).unwrap().token_ids()
588        );
589        // The marker in the untrusted segment is ordinary text: exactly one `<|im_end|>` id.
590        assert_eq!(
591            fast_ids.token_ids().iter().filter(|&&id| id == 474).count(),
592            1
593        );
594    }
595
596    #[test]
597    fn decode_matches_tiktoken_rs_and_skips_specials() {
598        let (reference, fast) = pair();
599        for text in corpus() {
600            let ids = fast.encode(&text).unwrap();
601            assert_eq!(
602                fast.decode(ids.token_ids(), false).unwrap(),
603                reference.decode(ids.token_ids(), false).unwrap(),
604                "{text:?}"
605            );
606        }
607        let ids = fast.encode("<|im_start|>user\nhi<|im_end|>").unwrap();
608        let kept = fast.decode(ids.token_ids(), false).unwrap();
609        let skipped = fast.decode(ids.token_ids(), true).unwrap();
610        assert!(kept.as_str().contains("<|im_end|>"));
611        assert!(!skipped.as_str().contains("<|im_end|>"));
612        assert_eq!(skipped, reference.decode(ids.token_ids(), true).unwrap());
613    }
614
615    #[test]
616    fn replacement_char_token_decodes_complete_like_tiktoken_rs() {
617        // Rank 468 is the bytes EF BF BD: a vocabulary token whose text is U+FFFD.
618        let (reference, fast) = pair();
619        let decoded = fast.decode(&[468], false).unwrap();
620        assert!(decoded.is_complete(), "{decoded:?}");
621        assert_eq!(decoded, reference.decode(&[468], false).unwrap());
622
623        // A truncated multi-byte sequence stays `Partial`.
624        let emoji = fast.encode("😀").unwrap();
625        let cut = &emoji.token_ids()[..emoji.token_ids().len() - 1];
626        let fast_cut = fast.decode(cut, false).unwrap();
627        assert!(fast_cut.is_partial(), "{fast_cut:?}");
628        assert_eq!(fast_cut, reference.decode(cut, false).unwrap());
629
630        // Unknown ids are skipped.
631        assert_eq!(fast.decode(&[468, 9_999_999], false).unwrap(), decoded);
632    }
633
634    #[test]
635    fn plain_text_prefix_cache_is_rejected() {
636        let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
637        let specials = fast.special_tokens().to_vec();
638        let error = crate::CachedTokenizer::new(std::sync::Arc::new(fast), specials, 1 << 20)
639            .err()
640            .expect("fastokens over tiktoken.model must not be prefix-cached")
641            .to_string();
642        assert!(error.contains("fastokens"), "{error}");
643    }
644
645    /// Fastokens 0.3.4 is consistent across this special-token boundary, but still
646    /// differs from tiktoken-rs on the Unicode case folding of `(?i:'s)` (`'ſ`).
647    /// This single case does not establish the full plain-text prefix-cache contract.
648    #[test]
649    fn unicode_contraction_is_consistent_across_special_token_boundary() {
650        let reference = TikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
651        let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
652        let special = "<|end_of_msg|>";
653        let suffix = " I'\u{17f}";
654        let full = format!("{special}{suffix}");
655
656        let ref_full = reference.encode(&full).unwrap().token_ids().to_vec();
657        let ref_suffix = reference.encode(suffix).unwrap().token_ids().to_vec();
658        assert_eq!(
659            ref_full[1..],
660            ref_suffix[..],
661            "tiktoken-rs must be self-consistent"
662        );
663
664        let fast_full = fast.encode(&full).unwrap().token_ids().to_vec();
665        let fast_suffix = fast.encode(suffix).unwrap().token_ids().to_vec();
666        assert_ne!(
667            fast_full, ref_full,
668            "revisit the known Unicode contraction mismatch"
669        );
670        assert_eq!(
671            fast_full[1..],
672            fast_suffix[..],
673            "splitting at this special-token boundary must preserve fastokens ids"
674        );
675    }
676}