Skip to main content

dynamo_tokenizers/
fastokens.rs

1// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Fastokens backend using the `fastokens` crate for high-performance BPE encoding.
5//!
6//! This module preserves the existing hybrid behavior: `fastokens` handles encoding and
7//! `HuggingFaceTokenizer` handles decoding. Both are loaded from the same `tokenizer.json`
8//! file.
9//!
10//! [`FastTikTokenTokenizer`] instead loads a bare `tiktoken.model` and decodes its ranks
11//! directly.
12
13use std::collections::HashSet;
14use std::path::Path;
15
16use rayon::prelude::*;
17use rustc_hash::FxHashMap;
18
19use super::{
20    EncodeSegment, Encoding, Error, Result, TokenIdType,
21    hf::HuggingFaceTokenizer,
22    tiktoken,
23    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
24};
25
26fn fast_encode(encoder: &fastokens::Tokenizer, input: &str) -> Result<Encoding> {
27    let ids = encoder
28        .encode(input)
29        .map_err(|e| Error::msg(format!("Fastokens encode error: {e}")))?;
30    Ok(Encoding::Sp(ids))
31}
32
33fn fast_encode_segments(
34    encoder: &fastokens::Tokenizer,
35    segments: &[EncodeSegment<'_>],
36) -> Result<Encoding> {
37    let segments: Vec<fastokens::EncodeSegment<'_>> = segments
38        .iter()
39        .map(|segment| fastokens::EncodeSegment {
40            text: segment.text,
41            allow_special: segment.allow_special,
42        })
43        .collect();
44    let ids = encoder
45        .encode_segments(&segments)
46        .map_err(|e| Error::msg(format!("Fastokens segmented encode error: {e}")))?;
47    Ok(Encoding::Sp(ids))
48}
49
50/// Hybrid tokenizer: fast BPE encoding via `fastokens`, decoding via HuggingFace.
51///
52/// Both backends are loaded from the same `tokenizer.json` file.
53pub struct FastTokenizer {
54    fast_encoder: fastokens::Tokenizer,
55    hf_decoder: HuggingFaceTokenizer,
56}
57
58impl FastTokenizer {
59    /// Load `tokenizer.json`, ignoring its `truncation` and `padding` blocks as Python
60    /// `transformers` does unless a call asks for them. `fastokens` would apply both on
61    /// every encode, and [`CachedTokenizer`](crate::CachedTokenizer) would then apply
62    /// them to each separately encoded piece instead of the whole input.
63    pub fn from_file(path: &str) -> Result<Self> {
64        let mut fast_encoder = fastokens::Tokenizer::from_file(Path::new(path))
65            .map_err(|e| Error::msg(format!("Error loading fastokens tokenizer: {e}")))?;
66        fast_encoder.set_truncation(None);
67        fast_encoder.set_padding(None);
68        let hf_decoder = HuggingFaceTokenizer::from_file(path)?;
69        Ok(Self {
70            fast_encoder,
71            hf_decoder,
72        })
73    }
74}
75
76impl Encoder for FastTokenizer {
77    fn encode(&self, input: &str) -> Result<Encoding> {
78        fast_encode(&self.fast_encoder, input)
79    }
80
81    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
82        inputs.par_iter().map(|input| self.encode(input)).collect()
83    }
84
85    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
86        fast_encode_segments(&self.fast_encoder, segments)
87    }
88}
89
90impl Decoder for FastTokenizer {
91    fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
92        self.hf_decoder
93            .has_unstable_suffix(token_ids, skip_special_tokens)
94    }
95
96    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
97        self.hf_decoder.decode(token_ids, skip_special_tokens)
98    }
99}
100
101impl Tokenizer for FastTokenizer {
102    fn validate_prefix_cache(&self) -> Result<()> {
103        Ok(())
104    }
105
106    // `fast_encoder` and `hf_decoder` are loaded from the same tokenizer.json,
107    // so the HF side's vocabulary introspection applies to both.
108    fn vocab_size(&self) -> Option<usize> {
109        self.hf_decoder.vocab_size()
110    }
111
112    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
113        self.hf_decoder.token_to_id(token)
114    }
115
116    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
117        self.hf_decoder.special_token_ids()
118    }
119
120    fn num_special_tokens_added(&self) -> Result<usize> {
121        Ok(0)
122    }
123}
124
125/// `fastokens` over a bare `tiktoken.model`, for checkpoints such as Kimi K2/K3 that ship no
126/// `tokenizer.json`. Loads the same ranks, regex, and special tokens as
127/// [`TikTokenTokenizer::from_file_auto`](crate::TikTokenTokenizer::from_file_auto).
128///
129/// Plain-text prefix caching ([`CachedTokenizer`](crate::CachedTokenizer)) is rejected for
130/// this backend pending validation of the full cache matrix. In fastokens 0.3.4, the
131/// known Unicode contraction case is consistent across a special-token boundary, but
132/// still differs from tiktoken-rs. Segmented encodes are unaffected by the cache gate:
133/// every segment is encoded on its own with or without a cache.
134pub struct FastTikTokenTokenizer {
135    inner: fastokens::Tokenizer,
136    /// Decoding joins raw bytes itself because `fastokens`' decoder returns a lossy `String`,
137    /// which cannot tell a vocabulary token ending in `EF BF BD` from a truncated sequence.
138    id_to_bytes: FxHashMap<u32, Vec<u8>>,
139    special_token_ids: HashSet<u32>,
140    special_tokens: Vec<String>,
141}
142
143impl FastTikTokenTokenizer {
144    /// Load a tiktoken model file, reading the BPE pattern from `config.json` and the special
145    /// tokens from `tokenizer_config.json` in the same directory.
146    pub fn from_file_auto(path: &str) -> Result<Self> {
147        let directory = Path::new(path)
148            .parent()
149            .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
150        let pattern = tiktoken::detect_bpe_pattern(directory)?;
151        let encoder = tiktoken::parse_tiktoken_file(path)?;
152        let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
153        let tiktoken::AddedTokens { tokens, skip_ids } =
154            tiktoken::load_added_tokens(directory, num_base_tokens)?;
155
156        let ranks: Vec<(Vec<u8>, u32)> = encoder.into_iter().collect();
157        let mut id_to_bytes: FxHashMap<u32, Vec<u8>> = ranks
158            .iter()
159            .map(|(bytes, rank)| (*rank, bytes.clone()))
160            .collect();
161        id_to_bytes.extend(
162            tokens
163                .iter()
164                .map(|(content, &id)| (id, content.as_bytes().to_vec())),
165        );
166        let special_token_strings = tiktoken::sorted_special_token_strings(&tokens);
167
168        let config =
169            fastokens::tiktoken::TiktokenConfig::new(pattern, tokens.into_iter().collect());
170        let inner = fastokens::Tokenizer::from_tiktoken_ranks(&ranks, config).map_err(|e| {
171            Error::msg(format!(
172                "Error loading fastokens tiktoken tokenizer from {path}: {e}"
173            ))
174        })?;
175        Ok(Self {
176            inner,
177            id_to_bytes,
178            special_token_ids: skip_ids,
179            special_tokens: special_token_strings,
180        })
181    }
182
183    /// Atomic special-token strings registered with the encoder, sorted; the boundary set
184    /// for [`CachedTokenizer`](crate::CachedTokenizer). Includes added tokens marked
185    /// `"special": false`, unlike `special_token_ids`.
186    pub fn special_tokens(&self) -> &[String] {
187        &self.special_tokens
188    }
189}
190
191impl Encoder for FastTikTokenTokenizer {
192    fn encode(&self, input: &str) -> Result<Encoding> {
193        fast_encode(&self.inner, input)
194    }
195
196    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
197        inputs.par_iter().map(|input| self.encode(input)).collect()
198    }
199
200    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
201        fast_encode_segments(&self.inner, segments)
202    }
203}
204
205impl Decoder for FastTikTokenTokenizer {
206    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
207        let mut bytes = Vec::new();
208        for id in token_ids {
209            if skip_special_tokens && self.special_token_ids.contains(id) {
210                continue;
211            }
212            if let Some(token) = self.id_to_bytes.get(id) {
213                bytes.extend_from_slice(token);
214            }
215        }
216        match String::from_utf8(bytes) {
217            Ok(text) => Ok(DecodeResult::Complete(text)),
218            Err(e) => Ok(DecodeResult::from_decoded(
219                String::from_utf8_lossy(e.as_bytes()).into_owned(),
220            )),
221        }
222    }
223}
224
225impl Tokenizer for FastTikTokenTokenizer {
226    fn validate_prefix_cache(&self) -> Result<()> {
227        Err(Error::msg(
228            "fastokens over tiktoken.model plain-text prefix caching is disabled pending \
229             validation of the full cache matrix",
230        ))
231    }
232
233    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
234        Ok(self.inner.token_to_id(token))
235    }
236
237    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
238        let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
239        ids.sort_unstable();
240        Ok(ids)
241    }
242
243    fn num_special_tokens_added(&self) -> Result<usize> {
244        Ok(0)
245    }
246}
247
248#[cfg(test)]
249mod tests {
250    use super::*;
251    use crate::{HuggingFaceTokenizer, TokenizerOptions};
252
253    // Minimal synthetic BPE tokenizer with no normalizer or post-processor --
254    // compatible with fastokens. Vocab covers: H,T,a,d,e,h,i,l,o,r,s,t,w + punctuation.
255    const TOKENIZER_PATH: &str = concat!(
256        env!("CARGO_MANIFEST_DIR"),
257        "/tests/data/minimal-bpe/tokenizer.json"
258    );
259    const SEGMENTED_TOKENIZER_PATH: &str = concat!(
260        env!("CARGO_MANIFEST_DIR"),
261        "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
262    );
263
264    #[test]
265    fn byte_fallback_stream_matches_full_decode() {
266        let tokenizer: crate::Tokenizer =
267            std::sync::Arc::new(FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap()).into();
268        for (pieces, skip) in [
269            (vec!["<0x61>", "<0xF5>"], false),
270            (vec!["<0x61>", "</s>", "<0xF5>"], false),
271            (vec!["<0x61>", "</s>", "<0xF5>"], true),
272        ] {
273            let ids: Vec<_> = pieces
274                .iter()
275                .map(|piece| tokenizer.token_to_id(piece).unwrap().unwrap())
276                .collect();
277            let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
278            let mut stream = tokenizer.decode_stream(&[], skip);
279            let mut actual = String::new();
280            for id in ids {
281                actual.push_str(&stream.step(id).unwrap().unwrap_or_default());
282            }
283            actual.push_str(&stream.finish().unwrap().unwrap_or_default());
284            assert_eq!(actual, expected, "{pieces:?}, skip={skip}");
285        }
286    }
287
288    #[test]
289    fn test_fast_encode_decode_roundtrip() {
290        let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
291        // Encode then decode: verifies both paths execute without error.
292        // With a null decoder, HF inserts spaces between tokens so exact equality
293        // is not expected here -- we just verify the operations succeed and produce
294        // non-empty results.
295        let text = "Hello, world!";
296        let encoding = tokenizer.encode(text).unwrap();
297        assert!(!encoding.token_ids().is_empty());
298        let decoded: String = tokenizer.decode(encoding.token_ids(), true).unwrap().into();
299        assert!(!decoded.is_empty());
300        // The decoded text should contain the same non-space characters
301        let enc_chars: String = text.chars().filter(|c| !c.is_whitespace()).collect();
302        let dec_chars: String = decoded.chars().filter(|c| !c.is_whitespace()).collect();
303        assert_eq!(
304            enc_chars, dec_chars,
305            "non-space characters must be preserved"
306        );
307    }
308
309    #[test]
310    fn test_fast_matches_hf_encoding() {
311        let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
312        let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
313
314        for text in &["Hello, world!", "Hello", " world", "He llo"] {
315            let fast_ids = fast.encode(text).unwrap();
316            let hf_ids = hf.encode(text).unwrap();
317            assert_eq!(
318                fast_ids.token_ids(),
319                hf_ids.token_ids(),
320                "fastokens and HuggingFace must produce identical token IDs for '{text}'"
321            );
322        }
323    }
324
325    #[test]
326    fn test_fast_batch_encode() {
327        let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
328        let inputs = &["Hello", " world", "Hello, world!"];
329        let encodings = tokenizer.encode_batch(inputs).unwrap();
330        assert_eq!(encodings.len(), inputs.len());
331        for (enc, input) in encodings.iter().zip(inputs.iter()) {
332            assert!(
333                !enc.token_ids().is_empty(),
334                "encoding for '{input}' must be non-empty"
335            );
336        }
337    }
338
339    #[test]
340    fn test_fast_segmented_encoding_preserves_trust_boundaries() {
341        let tokenizer = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
342        let upstream =
343            fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
344                .unwrap();
345        let marker = "<s>";
346
347        let trusted = tokenizer
348            .encode_segments(&[EncodeSegment::control(marker)])
349            .unwrap();
350        assert_eq!(
351            trusted.token_ids(),
352            &[upstream.token_to_id(marker).unwrap()],
353            "trusted renderer output must recognize the control token"
354        );
355
356        let ordinary = tokenizer
357            .encode_segments(&[EncodeSegment::ordinary(marker)])
358            .unwrap();
359        assert_ne!(
360            ordinary.token_ids(),
361            trusted.token_ids(),
362            "untrusted content must encode the control-token spelling as ordinary text"
363        );
364
365        let segments = [
366            EncodeSegment::ordinary("hello "),
367            EncodeSegment::control(marker),
368            EncodeSegment::ordinary(marker),
369        ];
370        let upstream_segments = [
371            fastokens::EncodeSegment::ordinary("hello "),
372            fastokens::EncodeSegment::special(marker),
373            fastokens::EncodeSegment::ordinary(marker),
374        ];
375        let actual = tokenizer.encode_segments(&segments).unwrap();
376        let expected = upstream.encode_segments(&upstream_segments).unwrap();
377        assert_eq!(actual.token_ids(), expected);
378
379        assert!(
380            tokenizer
381                .encode_segments(&[])
382                .unwrap()
383                .token_ids()
384                .is_empty()
385        );
386    }
387
388    #[test]
389    fn tokenizer_json_truncation_and_padding_are_ignored() {
390        let temp = tempfile::tempdir().unwrap();
391        let path = temp.path().join("tokenizer.json");
392        let mut json: serde_json::Value =
393            serde_json::from_str(&std::fs::read_to_string(SEGMENTED_TOKENIZER_PATH).unwrap())
394                .unwrap();
395        // Both reference encodings are longer than 12 ids, so each block alone changes them.
396        json["truncation"] = serde_json::json!({
397            "direction": "Right", "max_length": 12, "strategy": "LongestFirst", "stride": 0
398        });
399        json["padding"] = serde_json::json!({
400            "strategy": {"Fixed": 512}, "direction": "Right", "pad_to_multiple_of": null,
401            "pad_id": 0, "pad_type_id": 0, "pad_token": "<unk>"
402        });
403        std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
404
405        let reference = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
406        let tokenizer = FastTokenizer::from_file(path.to_str().unwrap()).unwrap();
407        let user = "hello world ".repeat(16);
408        let segments = [
409            EncodeSegment::control("<s>"),
410            EncodeSegment::ordinary(&user),
411            EncodeSegment::control("</s>"),
412        ];
413        assert_eq!(
414            tokenizer.encode(&user).unwrap().token_ids(),
415            reference.encode(&user).unwrap().token_ids()
416        );
417        assert_eq!(
418            tokenizer.encode_segments(&segments).unwrap().token_ids(),
419            reference.encode_segments(&segments).unwrap().token_ids()
420        );
421    }
422
423    #[test]
424    fn test_fast_with_decode_stream() {
425        use crate::Tokenizer as TokenizerWrapper;
426        use std::sync::Arc;
427
428        let tokenizer = Arc::new(FastTokenizer::from_file(TOKENIZER_PATH).unwrap());
429        let wrapper = TokenizerWrapper::from(tokenizer);
430
431        // Encode a prompt and a continuation, then step through the decode stream
432        let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
433        let continuation = ", world!";
434        let cont_ids = wrapper.encode(continuation).unwrap().token_ids().to_vec();
435
436        let mut stream = wrapper.decode_stream(&prompt_ids, true);
437        // Accumulate incremental chunks from decode_stream
438        let mut accumulated = String::new();
439        for id in &cont_ids {
440            if let Some(chunk) = stream.step(*id).unwrap() {
441                accumulated.push_str(&chunk);
442            }
443        }
444
445        // DecodeStream uses prompt tokens as context, so the expected text is
446        // decode(prompt + continuation) minus decode(prompt) -- not a bare
447        // decode(continuation) which lacks the surrounding context.
448        let mut all_ids = prompt_ids.clone();
449        all_ids.extend_from_slice(&cont_ids);
450        let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
451        let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
452        let expected = &full_text[prompt_text.len()..];
453        assert_eq!(
454            accumulated, expected,
455            "streamed chunks must equal context-aware decoded continuation"
456        );
457    }
458
459    #[test]
460    fn vocabulary_metadata_forwards_to_hf_decoder() {
461        let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
462        let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
463        assert_eq!(fast.vocab_size(), hf.vocab_size());
464        assert_eq!(
465            fast.token_to_id("Hello").unwrap(),
466            hf.token_to_id("Hello").unwrap()
467        );
468        assert_eq!(
469            fast.special_token_ids().unwrap(),
470            hf.special_token_ids().unwrap()
471        );
472    }
473
474    #[test]
475    fn special_token_accounting_matches_fast_encoder() {
476        let fast = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
477        let upstream =
478            fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
479                .unwrap();
480        let hf = HuggingFaceTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
481
482        assert_eq!(hf.num_special_tokens_added().unwrap(), 1);
483        assert_eq!(fast.num_special_tokens_added().unwrap(), 0);
484        let hf_with_special_tokens = hf.with_options(TokenizerOptions {
485            add_special_tokens: true,
486        });
487
488        for text in ["hello", "hello there"] {
489            let fast_ids = fast.encode(text).unwrap();
490            assert_eq!(
491                fast_ids.token_ids(),
492                upstream.encode(text).unwrap(),
493                "FastTokenizer must match the encoder that omits the HF post-processor"
494            );
495            assert_eq!(
496                hf_with_special_tokens
497                    .encode(text)
498                    .unwrap()
499                    .token_ids()
500                    .len(),
501                fast_ids.token_ids().len() + 1,
502                "the HF post-processor must add the BOS token FastTokenizer omits"
503            );
504        }
505    }
506}
507
508#[cfg(test)]
509mod tiktoken_parity_tests {
510    use super::*;
511    use crate::TikTokenTokenizer;
512
513    const TIKTOKEN_PATH: &str = concat!(
514        env!("CARGO_MANIFEST_DIR"),
515        "/tests/data/sample-models/mock-tiktoken-bpe/tiktoken.model"
516    );
517    /// All 256 byte tokens plus the merges ` I` and ` I'`, with the Kimi pattern: the
518    /// smallest vocabulary that exposes fastokens' Unicode contraction mismatch with tiktoken-rs.
519    const CONTRACTION_PATH: &str = concat!(
520        env!("CARGO_MANIFEST_DIR"),
521        "/tests/data/sample-models/mock-tiktoken-contraction/tiktoken.model"
522    );
523
524    fn pair() -> (TikTokenTokenizer, FastTikTokenTokenizer) {
525        let reference = TikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
526        let fast = FastTikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
527        (reference, fast)
528    }
529
530    fn corpus() -> Vec<String> {
531        vec![
532            String::new(),
533            "hello world".into(),
534            "Hello, World! 123 4567 89".into(),
535            "  leading and trailing  ".into(),
536            "tabs\tand\nnewlines\r\nmixed   spacing".into(),
537            "<|im_start|>user\nhi there<|im_end|><|im_start|>assistant\n".into(),
538            "a literal <|im_start|> inside plain text".into(),
539            "emoji 😀🚀 and café naïve Zürich".into(),
540            "北京 東京 mixed 中英文 text ソフトウェア".into(),
541            "Москва मुंबई العربية".into(),
542            "fn main() { println!(\"{}\", 42); } // code-ish ~!@#$%^&*()".into(),
543            "x".repeat(5000),
544            " ".repeat(300) + "after long whitespace",
545            "word ".repeat(400),
546        ]
547    }
548
549    #[test]
550    fn special_token_tables_match_tiktoken_rs() {
551        let (reference, fast) = pair();
552        assert_eq!(fast.special_tokens(), reference.special_tokens());
553        assert_eq!(
554            fast.special_token_ids().unwrap(),
555            reference.special_token_ids().unwrap()
556        );
557        assert_eq!(fast.token_to_id("<|im_end|>").unwrap(), Some(474));
558    }
559
560    #[test]
561    fn non_special_added_tokens_survive_skip_special_tokens() {
562        let source = Path::new(TIKTOKEN_PATH).parent().unwrap();
563        let dir = tempfile::tempdir().unwrap();
564        for file in ["tiktoken.model", "config.json"] {
565            std::fs::copy(source.join(file), dir.path().join(file)).unwrap();
566        }
567        let mut config: serde_json::Value = serde_json::from_str(
568            &std::fs::read_to_string(source.join("tokenizer_config.json")).unwrap(),
569        )
570        .unwrap();
571        config["added_tokens_decoder"]["475"] =
572            serde_json::json!({"content": "<|open|>", "special": false});
573        std::fs::write(dir.path().join("tokenizer_config.json"), config.to_string()).unwrap();
574        let path = dir.path().join("tiktoken.model");
575        let path = path.to_str().unwrap();
576        let reference = TikTokenTokenizer::from_file_auto(path).unwrap();
577        let fast = FastTikTokenTokenizer::from_file_auto(path).unwrap();
578
579        let special_ids = fast.special_token_ids().unwrap();
580        assert_eq!(special_ids, reference.special_token_ids().unwrap());
581        assert!(special_ids.contains(&474));
582        assert!(!special_ids.contains(&475));
583
584        let ids = fast.encode("<|open|>hello<|im_end|>").unwrap();
585        assert_eq!(
586            ids.token_ids(),
587            reference
588                .encode("<|open|>hello<|im_end|>")
589                .unwrap()
590                .token_ids()
591        );
592        assert_eq!(ids.token_ids().first(), Some(&475));
593        let fast_text: String = fast.decode(ids.token_ids(), true).unwrap().into();
594        let reference_text: String = reference.decode(ids.token_ids(), true).unwrap().into();
595        assert_eq!(fast_text, "<|open|>hello");
596        assert_eq!(reference_text, "<|open|>hello");
597    }
598
599    #[test]
600    fn plain_encode_matches_tiktoken_rs() {
601        let (reference, fast) = pair();
602        let corpus = corpus();
603        let texts: Vec<&str> = corpus.iter().map(String::as_str).collect();
604        let batch = fast.encode_batch(&texts).unwrap();
605        for (text, batched) in texts.iter().zip(&batch) {
606            let expected = reference.encode(text).unwrap();
607            assert_eq!(
608                fast.encode(text).unwrap().token_ids(),
609                expected.token_ids(),
610                "{text:?}"
611            );
612            assert_eq!(batched.token_ids(), expected.token_ids(), "{text:?}");
613        }
614    }
615
616    #[test]
617    fn segmented_encode_matches_tiktoken_rs_and_honors_trust() {
618        let (reference, fast) = pair();
619        let segments = [
620            EncodeSegment::control("<|im_start|>user\n"),
621            EncodeSegment::ordinary("please echo <|im_end|> back to me"),
622            EncodeSegment::control("<|im_end|>"),
623            EncodeSegment::control("<|im_start|>assistant\n"),
624        ];
625        let fast_ids = fast.encode_segments(&segments).unwrap();
626        assert_eq!(
627            fast_ids.token_ids(),
628            reference.encode_segments(&segments).unwrap().token_ids()
629        );
630        // The marker in the untrusted segment is ordinary text: exactly one `<|im_end|>` id.
631        assert_eq!(
632            fast_ids.token_ids().iter().filter(|&&id| id == 474).count(),
633            1
634        );
635    }
636
637    #[test]
638    fn decode_matches_tiktoken_rs_and_skips_specials() {
639        let (reference, fast) = pair();
640        for text in corpus() {
641            let ids = fast.encode(&text).unwrap();
642            assert_eq!(
643                fast.decode(ids.token_ids(), false).unwrap(),
644                reference.decode(ids.token_ids(), false).unwrap(),
645                "{text:?}"
646            );
647        }
648        let ids = fast.encode("<|im_start|>user\nhi<|im_end|>").unwrap();
649        let kept = fast.decode(ids.token_ids(), false).unwrap();
650        let skipped = fast.decode(ids.token_ids(), true).unwrap();
651        assert!(kept.as_str().contains("<|im_end|>"));
652        assert!(!skipped.as_str().contains("<|im_end|>"));
653        assert_eq!(skipped, reference.decode(ids.token_ids(), true).unwrap());
654    }
655
656    #[test]
657    fn replacement_char_token_decodes_complete_like_tiktoken_rs() {
658        // Rank 468 is the bytes EF BF BD: a vocabulary token whose text is U+FFFD.
659        let (reference, fast) = pair();
660        let decoded = fast.decode(&[468], false).unwrap();
661        assert!(decoded.is_complete(), "{decoded:?}");
662        assert_eq!(decoded, reference.decode(&[468], false).unwrap());
663
664        // A truncated multi-byte sequence stays `Partial`.
665        let emoji = fast.encode("😀").unwrap();
666        let cut = &emoji.token_ids()[..emoji.token_ids().len() - 1];
667        let fast_cut = fast.decode(cut, false).unwrap();
668        assert!(fast_cut.is_partial(), "{fast_cut:?}");
669        assert_eq!(fast_cut, reference.decode(cut, false).unwrap());
670
671        // Unknown ids are skipped.
672        assert_eq!(fast.decode(&[468, 9_999_999], false).unwrap(), decoded);
673    }
674
675    #[test]
676    fn plain_text_prefix_cache_is_rejected() {
677        let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
678        let specials = fast.special_tokens().to_vec();
679        let error = crate::CachedTokenizer::new(std::sync::Arc::new(fast), specials, 1 << 20)
680            .err()
681            .expect("fastokens over tiktoken.model must not be prefix-cached")
682            .to_string();
683        assert!(error.contains("fastokens"), "{error}");
684    }
685
686    /// Fastokens 0.3.4 is consistent across this special-token boundary, but still
687    /// differs from tiktoken-rs on the Unicode case folding of `(?i:'s)` (`'ſ`).
688    /// This single case does not establish the full plain-text prefix-cache contract.
689    #[test]
690    fn unicode_contraction_is_consistent_across_special_token_boundary() {
691        let reference = TikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
692        let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
693        let special = "<|end_of_msg|>";
694        let suffix = " I'\u{17f}";
695        let full = format!("{special}{suffix}");
696
697        let ref_full = reference.encode(&full).unwrap().token_ids().to_vec();
698        let ref_suffix = reference.encode(suffix).unwrap().token_ids().to_vec();
699        assert_eq!(
700            ref_full[1..],
701            ref_suffix[..],
702            "tiktoken-rs must be self-consistent"
703        );
704
705        let fast_full = fast.encode(&full).unwrap().token_ids().to_vec();
706        let fast_suffix = fast.encode(suffix).unwrap().token_ids().to_vec();
707        assert_ne!(
708            fast_full, ref_full,
709            "revisit the known Unicode contraction mismatch"
710        );
711        assert_eq!(
712            fast_full[1..],
713            fast_suffix[..],
714            "splitting at this special-token boundary must preserve fastokens ids"
715        );
716    }
717}