Skip to main content

dynamo_tokenizers/
fastokens.rs

1// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Fastokens backend using the `fastokens` crate for high-performance BPE encoding.
5//!
6//! This module preserves the existing hybrid behavior: `fastokens` handles encoding and
7//! `HuggingFaceTokenizer` handles decoding. Both are loaded from the same `tokenizer.json`
8//! file.
9//!
10//! [`FastTikTokenTokenizer`] instead loads a bare `tiktoken.model` and decodes its ranks
11//! directly.
12
13use std::collections::HashSet;
14use std::path::Path;
15
16use rayon::prelude::*;
17use rustc_hash::FxHashMap;
18
19use super::{
20    EncodeSegment, Encoding, Error, Result, TokenIdType,
21    hf::HuggingFaceTokenizer,
22    tiktoken,
23    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
24};
25
26fn fast_encode(encoder: &fastokens::Tokenizer, input: &str) -> Result<Encoding> {
27    let ids = encoder
28        .encode(input)
29        .map_err(|e| Error::msg(format!("Fastokens encode error: {e}")))?;
30    Ok(Encoding::Sp(ids))
31}
32
33fn fast_encode_segments(
34    encoder: &fastokens::Tokenizer,
35    segments: &[EncodeSegment<'_>],
36    control_ids: &HashSet<u32>,
37) -> Result<Encoding> {
38    // Each segment is encoded once and its ids joined, which is what
39    // `fastokens` does itself: neither tokenizer here keeps the truncation or
40    // padding that would apply to the joined ids.
41    let mut ids = Vec::new();
42    for segment in segments {
43        let start = ids.len();
44        let segment_ids: Vec<u32> = encoder
45            .encode_segments_unpadded(&[fastokens::EncodeSegment {
46                text: segment.text,
47                allow_special: segment.allow_special,
48            }])
49            .map_err(|e| Error::msg(format!("Fastokens segmented encode error: {e}")))?;
50        ids.extend(segment_ids);
51        if !segment.allow_special {
52            crate::reject_control_ids_in_ordinary_segment(&ids[start..], control_ids)?;
53        }
54    }
55    Ok(Encoding::Sp(ids))
56}
57
58/// Hybrid tokenizer: fast BPE encoding via `fastokens`, decoding via HuggingFace.
59///
60/// Both backends are loaded from the same `tokenizer.json` file.
61pub struct FastTokenizer {
62    fast_encoder: fastokens::Tokenizer,
63    hf_decoder: HuggingFaceTokenizer,
64}
65
66impl FastTokenizer {
67    /// Load `tokenizer.json`, ignoring its `truncation` and `padding` blocks as Python
68    /// `transformers` does unless a call asks for them. `fastokens` would apply both on
69    /// every encode, and [`CachedTokenizer`](crate::CachedTokenizer) would then apply
70    /// them to each separately encoded piece instead of the whole input.
71    pub fn from_file(path: &str) -> Result<Self> {
72        let mut fast_encoder = fastokens::Tokenizer::from_file(Path::new(path))
73            .map_err(|e| Error::msg(format!("Error loading fastokens tokenizer: {e}")))?;
74        fast_encoder.set_truncation(None);
75        fast_encoder.set_padding(None);
76        let hf_decoder = HuggingFaceTokenizer::from_file(path)?;
77        Ok(Self {
78            fast_encoder,
79            hf_decoder,
80        })
81    }
82}
83
84impl Encoder for FastTokenizer {
85    fn encode(&self, input: &str) -> Result<Encoding> {
86        fast_encode(&self.fast_encoder, input)
87    }
88
89    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
90        inputs.par_iter().map(|input| self.encode(input)).collect()
91    }
92
93    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
94        fast_encode_segments(&self.fast_encoder, segments, self.hf_decoder.added_ids())
95    }
96}
97
98impl Decoder for FastTokenizer {
99    fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
100        self.hf_decoder
101            .has_unstable_suffix(token_ids, skip_special_tokens)
102    }
103
104    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
105        self.hf_decoder.decode(token_ids, skip_special_tokens)
106    }
107}
108
109impl Tokenizer for FastTokenizer {
110    fn validate_prefix_cache(&self) -> Result<()> {
111        Ok(())
112    }
113
114    // `fastokens` runs one pipeline for `encode` and for a segment that allows
115    // special tokens.
116    fn validate_segment_prefix_cache(&self) -> Result<()> {
117        Ok(())
118    }
119
120    // `fast_encoder` and `hf_decoder` are loaded from the same tokenizer.json,
121    // so the HF side's vocabulary introspection applies to both.
122    fn vocab_size(&self) -> Option<usize> {
123        self.hf_decoder.vocab_size()
124    }
125
126    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
127        self.hf_decoder.token_to_id(token)
128    }
129
130    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
131        self.hf_decoder.special_token_ids()
132    }
133
134    fn num_special_tokens_added(&self) -> Result<usize> {
135        Ok(0)
136    }
137}
138
139/// `fastokens` over a bare `tiktoken.model`, for checkpoints such as Kimi K2/K3 that ship no
140/// `tokenizer.json`. Loads the same ranks, regex, and special tokens as
141/// [`TikTokenTokenizer::from_file_auto`](crate::TikTokenTokenizer::from_file_auto).
142///
143/// Plain-text prefix caching ([`CachedTokenizer`](crate::CachedTokenizer)) is rejected for
144/// this backend pending validation of the full cache matrix. In fastokens 0.3.4, the
145/// known Unicode contraction case is consistent across a special-token boundary, but
146/// still differs from tiktoken-rs. Segmented encodes are unaffected by the cache gate:
147/// every segment is encoded on its own with or without a cache.
148pub struct FastTikTokenTokenizer {
149    inner: fastokens::Tokenizer,
150    /// Decoding joins raw bytes itself because `fastokens`' decoder returns a lossy `String`,
151    /// which cannot tell a vocabulary token ending in `EF BF BD` from a truncated sequence.
152    id_to_bytes: FxHashMap<u32, Vec<u8>>,
153    special_token_ids: HashSet<u32>,
154    special_tokens: Vec<String>,
155    /// Every added-token id, `special: true` or not; see
156    /// [`crate::reject_control_ids_in_ordinary_segment`].
157    added_ids: HashSet<u32>,
158}
159
160impl FastTikTokenTokenizer {
161    /// Load a tiktoken model file, reading the BPE pattern from `config.json` and the special
162    /// tokens from `tokenizer_config.json` in the same directory.
163    pub fn from_file_auto(path: &str) -> Result<Self> {
164        let directory = Path::new(path)
165            .parent()
166            .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
167        let pattern = tiktoken::detect_bpe_pattern(directory)?;
168        let encoder = tiktoken::parse_tiktoken_file(path)?;
169        let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
170        let tiktoken::AddedTokens { tokens, skip_ids } =
171            tiktoken::load_added_tokens(directory, num_base_tokens)?;
172
173        let ranks: Vec<(Vec<u8>, u32)> = encoder.into_iter().collect();
174        let mut id_to_bytes: FxHashMap<u32, Vec<u8>> = ranks
175            .iter()
176            .map(|(bytes, rank)| (*rank, bytes.clone()))
177            .collect();
178        id_to_bytes.extend(
179            tokens
180                .iter()
181                .map(|(content, &id)| (id, content.as_bytes().to_vec())),
182        );
183        let special_token_strings = tiktoken::sorted_special_token_strings(&tokens);
184        let added_ids = tokens.values().copied().collect();
185
186        let config =
187            fastokens::tiktoken::TiktokenConfig::new(pattern, tokens.into_iter().collect());
188        let inner = fastokens::Tokenizer::from_tiktoken_ranks(&ranks, config).map_err(|e| {
189            Error::msg(format!(
190                "Error loading fastokens tiktoken tokenizer from {path}: {e}"
191            ))
192        })?;
193        Ok(Self {
194            inner,
195            id_to_bytes,
196            special_token_ids: skip_ids,
197            special_tokens: special_token_strings,
198            added_ids,
199        })
200    }
201
202    /// Atomic special-token strings registered with the encoder, sorted; the boundary set
203    /// for [`CachedTokenizer`](crate::CachedTokenizer). Includes added tokens marked
204    /// `"special": false`, unlike `special_token_ids`.
205    pub fn special_tokens(&self) -> &[String] {
206        &self.special_tokens
207    }
208}
209
210impl Encoder for FastTikTokenTokenizer {
211    fn encode(&self, input: &str) -> Result<Encoding> {
212        fast_encode(&self.inner, input)
213    }
214
215    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
216        inputs.par_iter().map(|input| self.encode(input)).collect()
217    }
218
219    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
220        fast_encode_segments(&self.inner, segments, &self.added_ids)
221    }
222}
223
224impl Decoder for FastTikTokenTokenizer {
225    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
226        let mut bytes = Vec::new();
227        for id in token_ids {
228            if skip_special_tokens && self.special_token_ids.contains(id) {
229                continue;
230            }
231            if let Some(token) = self.id_to_bytes.get(id) {
232                bytes.extend_from_slice(token);
233            }
234        }
235        match String::from_utf8(bytes) {
236            Ok(text) => Ok(DecodeResult::Complete(text)),
237            Err(e) => Ok(DecodeResult::from_decoded(
238                String::from_utf8_lossy(e.as_bytes()).into_owned(),
239            )),
240        }
241    }
242}
243
244impl Tokenizer for FastTikTokenTokenizer {
245    fn validate_prefix_cache(&self) -> Result<()> {
246        Err(Error::msg(
247            "fastokens over tiktoken.model plain-text prefix caching is disabled pending \
248             validation of the full cache matrix",
249        ))
250    }
251
252    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
253        Ok(self.inner.token_to_id(token))
254    }
255
256    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
257        let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
258        ids.sort_unstable();
259        Ok(ids)
260    }
261
262    fn num_special_tokens_added(&self) -> Result<usize> {
263        Ok(0)
264    }
265}
266
267#[cfg(test)]
268mod tests {
269    use super::*;
270    use crate::{HuggingFaceTokenizer, TokenizerOptions};
271
272    // Minimal synthetic BPE tokenizer with no normalizer or post-processor --
273    // compatible with fastokens. Vocab covers: H,T,a,d,e,h,i,l,o,r,s,t,w + punctuation.
274    const TOKENIZER_PATH: &str = concat!(
275        env!("CARGO_MANIFEST_DIR"),
276        "/tests/data/minimal-bpe/tokenizer.json"
277    );
278    const SEGMENTED_TOKENIZER_PATH: &str = concat!(
279        env!("CARGO_MANIFEST_DIR"),
280        "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
281    );
282
283    #[test]
284    fn byte_fallback_stream_matches_full_decode() {
285        let tokenizer: crate::Tokenizer =
286            std::sync::Arc::new(FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap()).into();
287        for (pieces, skip) in [
288            (vec!["<0x61>", "<0xF5>"], false),
289            (vec!["<0x61>", "</s>", "<0xF5>"], false),
290            (vec!["<0x61>", "</s>", "<0xF5>"], true),
291        ] {
292            let ids: Vec<_> = pieces
293                .iter()
294                .map(|piece| tokenizer.token_to_id(piece).unwrap().unwrap())
295                .collect();
296            let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
297            let mut stream = tokenizer.decode_stream(&[], skip);
298            let mut actual = String::new();
299            for id in ids {
300                actual.push_str(&stream.step(id).unwrap().unwrap_or_default());
301            }
302            actual.push_str(&stream.finish().unwrap().unwrap_or_default());
303            assert_eq!(actual, expected, "{pieces:?}, skip={skip}");
304        }
305    }
306
307    #[test]
308    fn test_fast_encode_decode_roundtrip() {
309        let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
310        // Encode then decode: verifies both paths execute without error.
311        // With a null decoder, HF inserts spaces between tokens so exact equality
312        // is not expected here -- we just verify the operations succeed and produce
313        // non-empty results.
314        let text = "Hello, world!";
315        let encoding = tokenizer.encode(text).unwrap();
316        assert!(!encoding.token_ids().is_empty());
317        let decoded: String = tokenizer.decode(encoding.token_ids(), true).unwrap().into();
318        assert!(!decoded.is_empty());
319        // The decoded text should contain the same non-space characters
320        let enc_chars: String = text.chars().filter(|c| !c.is_whitespace()).collect();
321        let dec_chars: String = decoded.chars().filter(|c| !c.is_whitespace()).collect();
322        assert_eq!(
323            enc_chars, dec_chars,
324            "non-space characters must be preserved"
325        );
326    }
327
328    #[test]
329    fn test_fast_matches_hf_encoding() {
330        let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
331        let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
332
333        for text in &["Hello, world!", "Hello", " world", "He llo"] {
334            let fast_ids = fast.encode(text).unwrap();
335            let hf_ids = hf.encode(text).unwrap();
336            assert_eq!(
337                fast_ids.token_ids(),
338                hf_ids.token_ids(),
339                "fastokens and HuggingFace must produce identical token IDs for '{text}'"
340            );
341        }
342    }
343
344    #[test]
345    fn test_fast_batch_encode() {
346        let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
347        let inputs = &["Hello", " world", "Hello, world!"];
348        let encodings = tokenizer.encode_batch(inputs).unwrap();
349        assert_eq!(encodings.len(), inputs.len());
350        for (enc, input) in encodings.iter().zip(inputs.iter()) {
351            assert!(
352                !enc.token_ids().is_empty(),
353                "encoding for '{input}' must be non-empty"
354            );
355        }
356    }
357
358    #[test]
359    fn test_fast_segmented_encoding_preserves_trust_boundaries() {
360        let tokenizer = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
361        let upstream =
362            fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
363                .unwrap();
364        let marker = "<s>";
365
366        let trusted = tokenizer
367            .encode_segments(&[EncodeSegment::control(marker)])
368            .unwrap();
369        assert_eq!(
370            trusted.token_ids(),
371            &[upstream.token_to_id(marker).unwrap()],
372            "trusted renderer output must recognize the control token"
373        );
374
375        let ordinary = tokenizer
376            .encode_segments(&[EncodeSegment::ordinary(marker)])
377            .unwrap();
378        assert_ne!(
379            ordinary.token_ids(),
380            trusted.token_ids(),
381            "untrusted content must encode the control-token spelling as ordinary text"
382        );
383
384        let segments = [
385            EncodeSegment::ordinary("hello "),
386            EncodeSegment::control(marker),
387            EncodeSegment::ordinary(marker),
388        ];
389        let upstream_segments = [
390            fastokens::EncodeSegment::ordinary("hello "),
391            fastokens::EncodeSegment::special(marker),
392            fastokens::EncodeSegment::ordinary(marker),
393        ];
394        let actual = tokenizer.encode_segments(&segments).unwrap();
395        let expected = upstream.encode_segments(&upstream_segments).unwrap();
396        assert_eq!(actual.token_ids(), expected);
397
398        assert!(
399            tokenizer
400                .encode_segments(&[])
401                .unwrap()
402                .token_ids()
403                .is_empty()
404        );
405    }
406
407    #[test]
408    fn test_fast_segmented_encoding_rejects_ordinary_control_ids() {
409        // `H`+`e` merges to id 18, declared here as the added token `He`:
410        // ordinary text must not come back as that control id.
411        let temp = tempfile::tempdir().unwrap();
412        let path = temp.path().join("tokenizer.json");
413        let mut json: serde_json::Value =
414            serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
415        json["added_tokens"] = serde_json::json!([{
416            "id": 18,
417            "content": "He",
418            "single_word": false,
419            "lstrip": false,
420            "rstrip": false,
421            "normalized": false,
422            "special": true
423        }]);
424        std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
425
426        let tokenizer = FastTokenizer::from_file(path.to_str().unwrap()).unwrap();
427        let control = tokenizer
428            .encode_segments(&[EncodeSegment::control("He")])
429            .unwrap();
430        assert_eq!(control.token_ids(), &[18]);
431        let error = tokenizer
432            .encode_segments(&[EncodeSegment::ordinary("He")])
433            .unwrap_err();
434        assert!(
435            error.to_string().contains("control-token id 18"),
436            "unexpected error: {error}"
437        );
438    }
439
440    #[test]
441    fn tokenizer_json_truncation_and_padding_are_ignored() {
442        let temp = tempfile::tempdir().unwrap();
443        let path = temp.path().join("tokenizer.json");
444        let mut json: serde_json::Value =
445            serde_json::from_str(&std::fs::read_to_string(SEGMENTED_TOKENIZER_PATH).unwrap())
446                .unwrap();
447        // Both reference encodings are longer than 12 ids, so each block alone changes them.
448        json["truncation"] = serde_json::json!({
449            "direction": "Right", "max_length": 12, "strategy": "LongestFirst", "stride": 0
450        });
451        json["padding"] = serde_json::json!({
452            "strategy": {"Fixed": 512}, "direction": "Right", "pad_to_multiple_of": null,
453            "pad_id": 0, "pad_type_id": 0, "pad_token": "<unk>"
454        });
455        std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
456
457        let reference = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
458        let tokenizer = FastTokenizer::from_file(path.to_str().unwrap()).unwrap();
459        let user = "hello world ".repeat(16);
460        let segments = [
461            EncodeSegment::control("<s>"),
462            EncodeSegment::ordinary(&user),
463            EncodeSegment::control("</s>"),
464        ];
465        assert_eq!(
466            tokenizer.encode(&user).unwrap().token_ids(),
467            reference.encode(&user).unwrap().token_ids()
468        );
469        assert_eq!(
470            tokenizer.encode_segments(&segments).unwrap().token_ids(),
471            reference.encode_segments(&segments).unwrap().token_ids()
472        );
473    }
474
475    #[test]
476    fn test_fast_with_decode_stream() {
477        use crate::Tokenizer as TokenizerWrapper;
478        use std::sync::Arc;
479
480        let tokenizer = Arc::new(FastTokenizer::from_file(TOKENIZER_PATH).unwrap());
481        let wrapper = TokenizerWrapper::from(tokenizer);
482
483        // Encode a prompt and a continuation, then step through the decode stream
484        let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
485        let continuation = ", world!";
486        let cont_ids = wrapper.encode(continuation).unwrap().token_ids().to_vec();
487
488        let mut stream = wrapper.decode_stream(&prompt_ids, true);
489        // Accumulate incremental chunks from decode_stream
490        let mut accumulated = String::new();
491        for id in &cont_ids {
492            if let Some(chunk) = stream.step(*id).unwrap() {
493                accumulated.push_str(&chunk);
494            }
495        }
496
497        // DecodeStream uses prompt tokens as context, so the expected text is
498        // decode(prompt + continuation) minus decode(prompt) -- not a bare
499        // decode(continuation) which lacks the surrounding context.
500        let mut all_ids = prompt_ids.clone();
501        all_ids.extend_from_slice(&cont_ids);
502        let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
503        let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
504        let expected = &full_text[prompt_text.len()..];
505        assert_eq!(
506            accumulated, expected,
507            "streamed chunks must equal context-aware decoded continuation"
508        );
509    }
510
511    #[test]
512    fn vocabulary_metadata_forwards_to_hf_decoder() {
513        let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
514        let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
515        assert_eq!(fast.vocab_size(), hf.vocab_size());
516        assert_eq!(
517            fast.token_to_id("Hello").unwrap(),
518            hf.token_to_id("Hello").unwrap()
519        );
520        assert_eq!(
521            fast.special_token_ids().unwrap(),
522            hf.special_token_ids().unwrap()
523        );
524    }
525
526    #[test]
527    fn special_token_accounting_matches_fast_encoder() {
528        let fast = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
529        let upstream =
530            fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
531                .unwrap();
532        let hf = HuggingFaceTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
533
534        assert_eq!(hf.num_special_tokens_added().unwrap(), 1);
535        assert_eq!(fast.num_special_tokens_added().unwrap(), 0);
536        let hf_with_special_tokens = hf.with_options(TokenizerOptions {
537            add_special_tokens: true,
538        });
539
540        for text in ["hello", "hello there"] {
541            let fast_ids = fast.encode(text).unwrap();
542            assert_eq!(
543                fast_ids.token_ids(),
544                upstream.encode(text).unwrap(),
545                "FastTokenizer must match the encoder that omits the HF post-processor"
546            );
547            assert_eq!(
548                hf_with_special_tokens
549                    .encode(text)
550                    .unwrap()
551                    .token_ids()
552                    .len(),
553                fast_ids.token_ids().len() + 1,
554                "the HF post-processor must add the BOS token FastTokenizer omits"
555            );
556        }
557    }
558}
559
560#[cfg(test)]
561mod tiktoken_parity_tests {
562    use super::*;
563    use crate::TikTokenTokenizer;
564
565    const TIKTOKEN_PATH: &str = concat!(
566        env!("CARGO_MANIFEST_DIR"),
567        "/tests/data/sample-models/mock-tiktoken-bpe/tiktoken.model"
568    );
569    /// All 256 byte tokens plus the merges ` I` and ` I'`, with the Kimi pattern: the
570    /// smallest vocabulary that exposes fastokens' Unicode contraction mismatch with tiktoken-rs.
571    const CONTRACTION_PATH: &str = concat!(
572        env!("CARGO_MANIFEST_DIR"),
573        "/tests/data/sample-models/mock-tiktoken-contraction/tiktoken.model"
574    );
575
576    fn pair() -> (TikTokenTokenizer, FastTikTokenTokenizer) {
577        let reference = TikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
578        let fast = FastTikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
579        (reference, fast)
580    }
581
582    fn corpus() -> Vec<String> {
583        vec![
584            String::new(),
585            "hello world".into(),
586            "Hello, World! 123 4567 89".into(),
587            "  leading and trailing  ".into(),
588            "tabs\tand\nnewlines\r\nmixed   spacing".into(),
589            "<|im_start|>user\nhi there<|im_end|><|im_start|>assistant\n".into(),
590            "a literal <|im_start|> inside plain text".into(),
591            "emoji 😀🚀 and café naïve Zürich".into(),
592            "北京 東京 mixed 中英文 text ソフトウェア".into(),
593            "Москва मुंबई العربية".into(),
594            "fn main() { println!(\"{}\", 42); } // code-ish ~!@#$%^&*()".into(),
595            "x".repeat(5000),
596            " ".repeat(300) + "after long whitespace",
597            "word ".repeat(400),
598        ]
599    }
600
601    #[test]
602    fn special_token_tables_match_tiktoken_rs() {
603        let (reference, fast) = pair();
604        assert_eq!(fast.special_tokens(), reference.special_tokens());
605        assert_eq!(
606            fast.special_token_ids().unwrap(),
607            reference.special_token_ids().unwrap()
608        );
609        assert_eq!(fast.token_to_id("<|im_end|>").unwrap(), Some(474));
610    }
611
612    #[test]
613    fn non_special_added_tokens_survive_skip_special_tokens() {
614        let source = Path::new(TIKTOKEN_PATH).parent().unwrap();
615        let dir = tempfile::tempdir().unwrap();
616        for file in ["tiktoken.model", "config.json"] {
617            std::fs::copy(source.join(file), dir.path().join(file)).unwrap();
618        }
619        let mut config: serde_json::Value = serde_json::from_str(
620            &std::fs::read_to_string(source.join("tokenizer_config.json")).unwrap(),
621        )
622        .unwrap();
623        config["added_tokens_decoder"]["475"] =
624            serde_json::json!({"content": "<|open|>", "special": false});
625        std::fs::write(dir.path().join("tokenizer_config.json"), config.to_string()).unwrap();
626        let path = dir.path().join("tiktoken.model");
627        let path = path.to_str().unwrap();
628        let reference = TikTokenTokenizer::from_file_auto(path).unwrap();
629        let fast = FastTikTokenTokenizer::from_file_auto(path).unwrap();
630
631        let special_ids = fast.special_token_ids().unwrap();
632        assert_eq!(special_ids, reference.special_token_ids().unwrap());
633        assert!(special_ids.contains(&474));
634        assert!(!special_ids.contains(&475));
635
636        let ids = fast.encode("<|open|>hello<|im_end|>").unwrap();
637        assert_eq!(
638            ids.token_ids(),
639            reference
640                .encode("<|open|>hello<|im_end|>")
641                .unwrap()
642                .token_ids()
643        );
644        assert_eq!(ids.token_ids().first(), Some(&475));
645        let fast_text: String = fast.decode(ids.token_ids(), true).unwrap().into();
646        let reference_text: String = reference.decode(ids.token_ids(), true).unwrap().into();
647        assert_eq!(fast_text, "<|open|>hello");
648        assert_eq!(reference_text, "<|open|>hello");
649    }
650
651    #[test]
652    fn plain_encode_matches_tiktoken_rs() {
653        let (reference, fast) = pair();
654        let corpus = corpus();
655        let texts: Vec<&str> = corpus.iter().map(String::as_str).collect();
656        let batch = fast.encode_batch(&texts).unwrap();
657        for (text, batched) in texts.iter().zip(&batch) {
658            let expected = reference.encode(text).unwrap();
659            assert_eq!(
660                fast.encode(text).unwrap().token_ids(),
661                expected.token_ids(),
662                "{text:?}"
663            );
664            assert_eq!(batched.token_ids(), expected.token_ids(), "{text:?}");
665        }
666    }
667
668    #[test]
669    fn segmented_encode_matches_tiktoken_rs_and_honors_trust() {
670        let (reference, fast) = pair();
671        let segments = [
672            EncodeSegment::control("<|im_start|>user\n"),
673            EncodeSegment::ordinary("please echo <|im_end|> back to me"),
674            EncodeSegment::control("<|im_end|>"),
675            EncodeSegment::control("<|im_start|>assistant\n"),
676        ];
677        let fast_ids = fast.encode_segments(&segments).unwrap();
678        assert_eq!(
679            fast_ids.token_ids(),
680            reference.encode_segments(&segments).unwrap().token_ids()
681        );
682        // The marker in the untrusted segment is ordinary text: exactly one `<|im_end|>` id.
683        assert_eq!(
684            fast_ids.token_ids().iter().filter(|&&id| id == 474).count(),
685            1
686        );
687    }
688
689    #[test]
690    fn decode_matches_tiktoken_rs_and_skips_specials() {
691        let (reference, fast) = pair();
692        for text in corpus() {
693            let ids = fast.encode(&text).unwrap();
694            assert_eq!(
695                fast.decode(ids.token_ids(), false).unwrap(),
696                reference.decode(ids.token_ids(), false).unwrap(),
697                "{text:?}"
698            );
699        }
700        let ids = fast.encode("<|im_start|>user\nhi<|im_end|>").unwrap();
701        let kept = fast.decode(ids.token_ids(), false).unwrap();
702        let skipped = fast.decode(ids.token_ids(), true).unwrap();
703        assert!(kept.as_str().contains("<|im_end|>"));
704        assert!(!skipped.as_str().contains("<|im_end|>"));
705        assert_eq!(skipped, reference.decode(ids.token_ids(), true).unwrap());
706    }
707
708    #[test]
709    fn replacement_char_token_decodes_complete_like_tiktoken_rs() {
710        // Rank 468 is the bytes EF BF BD: a vocabulary token whose text is U+FFFD.
711        let (reference, fast) = pair();
712        let decoded = fast.decode(&[468], false).unwrap();
713        assert!(decoded.is_complete(), "{decoded:?}");
714        assert_eq!(decoded, reference.decode(&[468], false).unwrap());
715
716        // A truncated multi-byte sequence stays `Partial`.
717        let emoji = fast.encode("😀").unwrap();
718        let cut = &emoji.token_ids()[..emoji.token_ids().len() - 1];
719        let fast_cut = fast.decode(cut, false).unwrap();
720        assert!(fast_cut.is_partial(), "{fast_cut:?}");
721        assert_eq!(fast_cut, reference.decode(cut, false).unwrap());
722
723        // Unknown ids are skipped.
724        assert_eq!(fast.decode(&[468, 9_999_999], false).unwrap(), decoded);
725    }
726
727    #[test]
728    fn plain_text_prefix_cache_is_rejected() {
729        let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
730        let specials = fast.special_tokens().to_vec();
731        let error = crate::CachedTokenizer::new(std::sync::Arc::new(fast), specials, 1 << 20)
732            .err()
733            .expect("fastokens over tiktoken.model must not be prefix-cached")
734            .to_string();
735        assert!(error.contains("fastokens"), "{error}");
736    }
737
738    /// Fastokens 0.3.4 is consistent across this special-token boundary, but still
739    /// differs from tiktoken-rs on the Unicode case folding of `(?i:'s)` (`'ſ`).
740    /// This single case does not establish the full plain-text prefix-cache contract.
741    #[test]
742    fn unicode_contraction_is_consistent_across_special_token_boundary() {
743        let reference = TikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
744        let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
745        let special = "<|end_of_msg|>";
746        let suffix = " I'\u{17f}";
747        let full = format!("{special}{suffix}");
748
749        let ref_full = reference.encode(&full).unwrap().token_ids().to_vec();
750        let ref_suffix = reference.encode(suffix).unwrap().token_ids().to_vec();
751        assert_eq!(
752            ref_full[1..],
753            ref_suffix[..],
754            "tiktoken-rs must be self-consistent"
755        );
756
757        let fast_full = fast.encode(&full).unwrap().token_ids().to_vec();
758        let fast_suffix = fast.encode(suffix).unwrap().token_ids().to_vec();
759        assert_ne!(
760            fast_full, ref_full,
761            "revisit the known Unicode contraction mismatch"
762        );
763        assert_eq!(
764            fast_full[1..],
765            fast_suffix[..],
766            "splitting at this special-token boundary must preserve fastokens ids"
767        );
768    }
769}