Skip to main content

dynamo_tokenizers/
tekken.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Mistral Tekken tokenizer (`tekken.json`).
5//!
6//! Tekken is a tiktoken-style byte-level BPE. The file stores the BPE ranks,
7//! the split regex, and the special tokens. Special tokens own ids
8//! `0..num_special_tokens`; an ordinary BPE token with rank `r` has id
9//! `r + num_special_tokens`. This mirrors mistral-common's `Tekkenizer`.
10//!
11//! Plain `encode` never recognizes special-token text, matching
12//! mistral-common, which encodes message text as ordinary BPE and inserts
13//! structural tokens by id. Chat prompts must therefore arrive as segments
14//! from a renderer that marks its control tokens ([`EncodeSegment::control`]);
15//! a text-only Jinja prompt would encode `<s>[INST]` as plain text.
16
17use std::path::Path;
18
19use aho_corasick::{AhoCorasick, MatchKind};
20use anyhow::Context as _;
21use base64::Engine as _;
22use fancy_regex::Regex;
23use rayon::prelude::*;
24use rustc_hash::FxHashMap;
25use serde::Deserialize;
26use tiktoken_rs::{CoreBPE, byte_pair_split};
27
28use super::{
29    EncodeSegment, Encoding, Error, Result, TokenIdType, TokenizerOptions,
30    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
31};
32
33/// Special tokens kept by `decode(skip_special_tokens = true)`.
34///
35/// Tool-call and reasoning parsers need these markers in the decoded text.
36/// Upstream vLLM's `MistralTokenizer.convert_ids_to_tokens` keeps the same set
37/// when skipping special tokens.
38const PARSER_MARKERS: [&str; 4] = ["[TOOL_CALLS]", "[ARGS]", "[THINK]", "[/THINK]"];
39
40/// Released tekken files reserve 1,000 special tokens. The bound keeps a
41/// malformed file from allocating billions of `<SPECIAL_n>` fillers.
42const MAX_SPECIAL_TOKENS: usize = 1 << 16;
43
44/// Byte-level BPE needs every single byte as a token.
45const BYTE_TOKENS: usize = 256;
46
47#[derive(Deserialize)]
48struct TekkenFile {
49    config: TekkenConfig,
50    vocab: Vec<TekkenTokenInfo>,
51    special_tokens: Option<Vec<TekkenSpecialToken>>,
52    model_settings_builder: Option<serde_json::Value>,
53}
54
55#[derive(Deserialize)]
56struct TekkenConfig {
57    pattern: String,
58    default_vocab_size: usize,
59    default_num_special_tokens: usize,
60    version: String,
61}
62
63#[derive(Deserialize)]
64struct TekkenTokenInfo {
65    rank: usize,
66    token_bytes: String,
67}
68
69#[derive(Deserialize)]
70struct TekkenSpecialToken {
71    rank: usize,
72    token_str: String,
73}
74
75pub struct TekkenTokenizer {
76    bpe: CoreBPE,
77    pattern: Regex,
78    special_token_matcher: AhoCorasick,
79    ordinary_token_ids: FxHashMap<Vec<u8>, TokenIdType>,
80    num_special_tokens: u32,
81    vocab_size: u32,
82    special_token_ids: FxHashMap<String, TokenIdType>,
83    kept_on_skip: Vec<TokenIdType>,
84    version: String,
85    model_settings_builder: Option<serde_json::Value>,
86}
87
88impl TekkenTokenizer {
89    /// Load a Mistral `tekken.json` tokenizer file.
90    pub fn from_file(path: &str) -> Result<Self> {
91        let file: TekkenFile = {
92            let contents = std::fs::read(path)
93                .with_context(|| format!("Failed to read tekken file '{path}'"))?;
94            serde_json::from_slice(&contents)
95                .with_context(|| format!("Failed to parse tekken file '{path}'"))?
96        };
97        Self::from_parts(file)
98    }
99
100    fn from_parts(file: TekkenFile) -> Result<Self> {
101        let TekkenConfig {
102            pattern,
103            default_vocab_size,
104            default_num_special_tokens,
105            version,
106        } = file.config;
107        let vocab_size = u32::try_from(default_vocab_size)
108            .context("tekken default_vocab_size does not fit a token id")?;
109        let num_special_tokens = u32::try_from(default_num_special_tokens)
110            .context("tekken default_num_special_tokens does not fit a token id")?;
111        if default_num_special_tokens > MAX_SPECIAL_TOKENS {
112            anyhow::bail!(
113                "tekken reserves {default_num_special_tokens} special tokens, more than the supported {MAX_SPECIAL_TOKENS}"
114            );
115        }
116
117        let Some(specials) = file.special_tokens else {
118            anyhow::bail!(
119                "tekken {version} file has no special_tokens; only files that list them are supported"
120            );
121        };
122        if specials.len() > default_num_special_tokens {
123            anyhow::bail!(
124                "tekken file lists {} special tokens but reserves only {default_num_special_tokens}",
125                specials.len()
126            );
127        }
128        let Some(inner_vocab_size) = default_vocab_size.checked_sub(default_num_special_tokens)
129        else {
130            anyhow::bail!("tekken default_vocab_size is smaller than its special tokens");
131        };
132        // Byte-pair merging requires every byte to have a token.
133        if inner_vocab_size < BYTE_TOKENS {
134            anyhow::bail!(
135                "tekken keeps {inner_vocab_size} ordinary tokens, fewer than the {BYTE_TOKENS} single bytes"
136            );
137        }
138        if file.vocab.len() < inner_vocab_size {
139            anyhow::bail!(
140                "tekken vocab has {} entries, fewer than the {inner_vocab_size} the config requires",
141                file.vocab.len()
142            );
143        }
144
145        // Ranks the file leaves unlisted are filled the way mistral-common does.
146        let mut special_tokens: Vec<String> = (0..default_num_special_tokens)
147            .map(|id| format!("<SPECIAL_{id}>"))
148            .collect();
149        for (index, token) in specials.into_iter().enumerate() {
150            if token.rank != index {
151                anyhow::bail!(
152                    "tekken special token '{}' has rank {}, expected {index}",
153                    token.token_str,
154                    token.rank
155                );
156            }
157            // An empty special token would make CoreBPE's special-token regex
158            // match the empty string and loop forever.
159            if token.token_str.is_empty() {
160                anyhow::bail!("tekken special token {index} is empty");
161            }
162            special_tokens[index] = token.token_str;
163        }
164
165        let mut special_token_ids = FxHashMap::default();
166        for (id, token) in (0..num_special_tokens).zip(special_tokens) {
167            if let Some(previous) = special_token_ids.insert(token, id) {
168                anyhow::bail!("tekken special token ids {previous} and {id} share one string");
169            }
170        }
171
172        let engine = base64::engine::general_purpose::STANDARD;
173        let mut ordinary_token_ids = FxHashMap::default();
174        for (rank, token) in (0..).zip(file.vocab.into_iter().take(inner_vocab_size)) {
175            if token.rank != rank as usize {
176                anyhow::bail!("tekken vocab entry {rank} has rank {}", token.rank);
177            }
178            let bytes = engine
179                .decode(&token.token_bytes)
180                .with_context(|| format!("Invalid base64 in tekken vocab entry {rank}"))?;
181            // mistral-common requires ranks 0..256 to be the single bytes in order.
182            if (rank as usize) < BYTE_TOKENS && bytes != [rank as u8] {
183                anyhow::bail!("tekken vocab entry {rank} is not the single byte {rank}");
184            }
185            // Shifting every rank preserves merge priority and emits tekken ids.
186            if ordinary_token_ids
187                .insert(bytes, num_special_tokens + rank)
188                .is_some()
189            {
190                anyhow::bail!("tekken vocab entry {rank} duplicates an earlier token");
191            }
192        }
193
194        let pattern = Regex::new(&pattern).context("Error creating tekken split regex")?;
195        // CoreBPE's encoder unwraps regex failures. Keep the same ranks and
196        // pattern for fallible encoding, and use CoreBPE only for decoding.
197        let bpe = CoreBPE::new(
198            ordinary_token_ids.clone(),
199            special_token_ids.clone(),
200            pattern.as_str(),
201        )
202        .map_err(|err| Error::msg(format!("Error creating tekken BPE: {err}")))?;
203        let special_token_matcher = AhoCorasick::builder()
204            .match_kind(MatchKind::LeftmostFirst)
205            .build(special_token_ids.keys())
206            .context("Error creating tekken special-token matcher")?;
207
208        let kept_on_skip = PARSER_MARKERS
209            .iter()
210            .filter_map(|marker| special_token_ids.get(*marker).copied())
211            .collect();
212
213        Ok(Self {
214            bpe,
215            pattern,
216            special_token_matcher,
217            ordinary_token_ids,
218            num_special_tokens,
219            vocab_size,
220            special_token_ids,
221            kept_on_skip,
222            version,
223            model_settings_builder: file.model_settings_builder,
224        })
225    }
226
227    /// Whether `path` names a Mistral tekken tokenizer file (`tekken*.json`,
228    /// such as `tekken.json` or mistral-common's versioned `tekken_240911.json`).
229    pub fn is_tekken_file(path: &Path) -> bool {
230        path.file_name()
231            .and_then(std::ffi::OsStr::to_str)
232            .is_some_and(|name| name.starts_with("tekken") && name.ends_with(".json"))
233    }
234
235    /// The tokenizer format version, such as `v15`.
236    pub fn version(&self) -> &str {
237        &self.version
238    }
239
240    /// The numeric part of [`Self::version`], such as `15` for `v15`.
241    pub fn version_number(&self) -> Option<u32> {
242        self.version.strip_prefix('v')?.parse().ok()
243    }
244
245    /// The raw `model_settings_builder` section (v15+), which sets the
246    /// allowed values and defaults of `[MODEL_SETTINGS]` prompt fields.
247    pub fn model_settings_builder(&self) -> Option<&serde_json::Value> {
248        self.model_settings_builder.as_ref()
249    }
250
251    /// Reject ids outside the vocabulary before decoding.
252    ///
253    /// `CoreBPE::_decode_native_and_split` panics on an unknown id. Every id
254    /// below `vocab_size` is known: special ids `0..N` are filled densely and
255    /// ordinary ranks are contiguous and unique, as `from_parts` checks.
256    fn validate_ids(&self, token_ids: &[TokenIdType]) -> Result<()> {
257        match token_ids.iter().find(|&&id| id >= self.vocab_size) {
258            Some(id) => anyhow::bail!(
259                "token id {id} is outside the tekken vocabulary of {}",
260                self.vocab_size
261            ),
262            None => Ok(()),
263        }
264    }
265
266    fn encode_ordinary_into(&self, input: &str, token_ids: &mut Vec<u32>) -> Result<()> {
267        for piece in self.pattern.find_iter(input) {
268            let piece = piece.context("Failed to match tekken split regex")?;
269            let bytes = piece.as_str().as_bytes();
270            if let Some(&id) = self.ordinary_token_ids.get(bytes) {
271                token_ids.push(id);
272            } else {
273                anyhow::ensure!(
274                    !bytes.is_empty(),
275                    "tekken split regex matched an empty string"
276                );
277                // Every one-byte token exists, so a missing whole piece has at
278                // least two bytes. Use the same merge primitive as CoreBPE.
279                token_ids.extend(
280                    byte_pair_split(bytes, &self.ordinary_token_ids)
281                        .into_iter()
282                        .map(|piece| self.ordinary_token_ids[piece]),
283                );
284            }
285        }
286        Ok(())
287    }
288
289    fn encode_special_into(&self, input: &str, token_ids: &mut Vec<u32>) -> Result<()> {
290        let mut end = 0;
291        for matched in self.special_token_matcher.find_iter(input) {
292            self.encode_ordinary_into(&input[end..matched.start()], token_ids)?;
293            let id = self.special_token_ids[&input[matched.start()..matched.end()]];
294            token_ids.push(id);
295            end = matched.end();
296        }
297        self.encode_ordinary_into(&input[end..], token_ids)
298    }
299}
300
301impl Encoder for TekkenTokenizer {
302    fn encode(&self, input: &str) -> Result<Encoding> {
303        let mut token_ids = Vec::new();
304        self.encode_ordinary_into(input, &mut token_ids)?;
305        Ok(Encoding::Sp(token_ids))
306    }
307
308    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
309        inputs.par_iter().map(|input| self.encode(input)).collect()
310    }
311
312    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
313        let mut token_ids = Vec::new();
314        for segment in segments {
315            if !segment.allow_special {
316                self.encode_ordinary_into(segment.text, &mut token_ids)?;
317            } else if let Some(&id) = self.special_token_ids.get(segment.text) {
318                // Renderers emit one control segment per marker.
319                token_ids.push(id);
320            } else {
321                self.encode_special_into(segment.text, &mut token_ids)?;
322            }
323        }
324        Ok(Encoding::Sp(token_ids))
325    }
326}
327
328impl Decoder for TekkenTokenizer {
329    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
330        self.validate_ids(token_ids)?;
331        let ids: Vec<u32> = if skip_special_tokens {
332            token_ids
333                .iter()
334                .filter(|&&id| id >= self.num_special_tokens || self.kept_on_skip.contains(&id))
335                .copied()
336                .collect()
337        } else {
338            token_ids.to_vec()
339        };
340
341        // Same strategy as the tiktoken backend: strict UTF-8 first, then a
342        // lossy fallback whose trailing U+FFFD marks an incomplete byte run.
343        let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
344        match String::from_utf8(bytes) {
345            Ok(text) => Ok(DecodeResult::Complete(text)),
346            Err(e) => {
347                let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
348                Ok(DecodeResult::from_decoded(text))
349            }
350        }
351    }
352}
353
354impl Tokenizer for TekkenTokenizer {
355    // `validate_prefix_cache` keeps its default error: the prefix cache splits
356    // text at special-token strings, but `encode` treats them as ordinary text.
357
358    /// `add_special_tokens` is not applied: `encode` never adds `<s>`, which
359    /// the Mistral renderer emits itself, as mistral-common does.
360    fn with_options(self, _options: TokenizerOptions) -> Self {
361        self
362    }
363
364    fn vocab_size(&self) -> Option<usize> {
365        Some(self.vocab_size as usize)
366    }
367
368    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
369        if let Some(&id) = self.special_token_ids.get(token) {
370            return Ok(Some(id));
371        }
372        // A string that BPE encodes to exactly one id is that token.
373        Ok(match self.encode(token)?.token_ids() {
374            &[id] => Some(id),
375            _ => None,
376        })
377    }
378
379    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
380        Ok((0..self.num_special_tokens).collect())
381    }
382
383    // Tekken's `encode` never adds BOS/EOS; renderers emit `<s>` explicitly.
384    fn num_special_tokens_added(&self) -> Result<usize> {
385        Ok(0)
386    }
387}
388
389#[cfg(test)]
390mod tests {
391    use super::*;
392    use crate::DecodeStream;
393    use std::sync::Arc;
394
395    const FIXTURE: &str = concat!(
396        env!("CARGO_MANIFEST_DIR"),
397        "/tests/data/sample-models/mock-tekken/tekken.json"
398    );
399    const EXPECTED_IDS: &str = concat!(
400        env!("CARGO_MANIFEST_DIR"),
401        "/tests/data/sample-models/mock-tekken/expected_ids.json"
402    );
403
404    #[derive(Deserialize)]
405    struct ExpectedIds {
406        text: String,
407        ids: Vec<TokenIdType>,
408    }
409
410    fn id_of(tokenizer: &TekkenTokenizer, token: &str) -> Option<TokenIdType> {
411        tokenizer.token_to_id(token).unwrap()
412    }
413
414    fn fixture() -> TekkenTokenizer {
415        TekkenTokenizer::from_file(FIXTURE).unwrap()
416    }
417
418    #[test]
419    fn encode_matches_mistral_common() {
420        let tokenizer = fixture();
421        let expected: Vec<ExpectedIds> =
422            serde_json::from_str(&std::fs::read_to_string(EXPECTED_IDS).unwrap()).unwrap();
423        assert!(!expected.is_empty());
424        for case in expected {
425            let ids = tokenizer.encode(&case.text).unwrap();
426            assert_eq!(ids.token_ids(), case.ids.as_slice(), "text {:?}", case.text);
427            let decoded: String = tokenizer.decode(&case.ids, false).unwrap().into();
428            assert_eq!(decoded, case.text);
429        }
430    }
431
432    #[test]
433    fn metadata_comes_from_the_file() {
434        let tokenizer = fixture();
435        assert_eq!(tokenizer.version(), "v15");
436        assert_eq!(tokenizer.version_number(), Some(15));
437        assert_eq!(tokenizer.vocab_size(), Some(1040));
438        assert_eq!(id_of(&tokenizer, "<s>"), Some(1));
439        assert_eq!(id_of(&tokenizer, "</s>"), Some(2));
440        assert_eq!(id_of(&tokenizer, "[TOOL_CALLS]"), Some(9));
441        assert_eq!(id_of(&tokenizer, "[THINK]"), Some(34));
442        // Ranks the file leaves unnamed become numbered fillers.
443        assert_eq!(id_of(&tokenizer, "<SPECIAL_39>"), Some(39));
444        assert_eq!(tokenizer.special_token_ids().unwrap().len(), 40);
445        assert_eq!(tokenizer.token_to_id("[ARGS]").unwrap(), Some(32));
446        assert_eq!(
447            tokenizer.token_to_id("a").unwrap(),
448            Some(40 + u32::from(b'a'))
449        );
450        assert_eq!(tokenizer.token_to_id("not-a-token-xyz").unwrap(), None);
451        let builder = tokenizer.model_settings_builder().unwrap();
452        assert_eq!(builder["reasoning_effort"]["default"], "none");
453    }
454
455    #[test]
456    fn special_token_text_is_ordinary_unless_marked_control() {
457        let tokenizer = fixture();
458        let inst = id_of(&tokenizer, "[INST]").unwrap();
459
460        let plain = tokenizer.encode("[INST]").unwrap();
461        assert!(!plain.token_ids().contains(&inst));
462
463        let segments = tokenizer
464            .encode_segments(&[
465                EncodeSegment::control("<s>[INST]"),
466                EncodeSegment::ordinary("hi [INST]"),
467                EncodeSegment::control("[/INST]"),
468            ])
469            .unwrap();
470        let ids = segments.token_ids();
471        assert_eq!(ids[..2], [1, inst]);
472        assert_eq!(*ids.last().unwrap(), 4);
473        assert_eq!(ids.iter().filter(|&&id| id == inst).count(), 1);
474    }
475
476    #[test]
477    fn skipping_special_tokens_keeps_parser_markers() {
478        let tokenizer = fixture();
479        let id = |token: &str| id_of(&tokenizer, token).unwrap();
480        let mut ids = vec![id("<s>"), id("[THINK]")];
481        ids.extend(tokenizer.encode("plan").unwrap().token_ids());
482        ids.extend([id("[/THINK]"), id("[TOOL_CALLS]")]);
483        ids.extend(tokenizer.encode("f").unwrap().token_ids());
484        ids.push(id("[ARGS]"));
485        ids.extend(tokenizer.encode("{}").unwrap().token_ids());
486        ids.push(id("</s>"));
487
488        let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
489        assert_eq!(skipped, "[THINK]plan[/THINK][TOOL_CALLS]f[ARGS]{}");
490        let kept: String = tokenizer.decode(&ids, false).unwrap().into();
491        assert_eq!(kept, "<s>[THINK]plan[/THINK][TOOL_CALLS]f[ARGS]{}</s>");
492    }
493
494    #[test]
495    fn decode_rejects_ids_outside_the_vocabulary() {
496        let tokenizer = fixture();
497        assert!(tokenizer.decode(&[1040], false).is_err());
498    }
499
500    #[test]
501    fn decode_stream_reassembles_byte_fallback_characters() {
502        let tokenizer = fixture();
503        let ids = tokenizer.encode("北京 😀").unwrap().token_ids().to_vec();
504        let tokenizer: Arc<dyn Tokenizer> = Arc::new(tokenizer);
505        let mut stream = DecodeStream::new(tokenizer, &[], false);
506        let mut text = String::new();
507        for id in ids {
508            if let Some(chunk) = stream.step(id).unwrap() {
509                assert!(!chunk.contains('\u{FFFD}'), "chunk {chunk:?}");
510                text.push_str(&chunk);
511            }
512        }
513        if let Some(chunk) = stream.finish().unwrap() {
514            text.push_str(&chunk);
515        }
516        assert_eq!(text, "北京 😀");
517    }
518
519    #[test]
520    fn file_dispatch_selects_tekken() {
521        let tokenizer = crate::create_tokenizer_from_file(FIXTURE).unwrap();
522        assert_eq!(tokenizer.vocab_size(), Some(1040));
523        let ids = tokenizer.encode("Hello, world!").unwrap();
524        assert_eq!(
525            ids.token_ids(),
526            fixture().encode("Hello, world!").unwrap().token_ids()
527        );
528    }
529
530    #[test]
531    fn control_segments_match_special_token_matching() {
532        let tokenizer = fixture();
533        let fast = tokenizer
534            .encode_segments(&[EncodeSegment::control("[INST]")])
535            .unwrap();
536        let mixed = tokenizer
537            .encode_segments(&[EncodeSegment::control("[INST][/INST]")])
538            .unwrap();
539        assert_eq!(fast.token_ids(), [3]);
540        assert_eq!(mixed.token_ids(), [3, 4]);
541    }
542
543    #[test]
544    fn fallible_encoding_matches_core_bpe() {
545        let tokenizer = fixture();
546        for text in [
547            "We're in Beijing 北京 😀\nDon't split café's accents.",
548            "'re\u{0301} I'M 123\r\n\t  /[INST]/ [ARGS]{}",
549            "<s>[INST]hi[/INST][TOOL_CALLS]f[ARGS]{}[THINK]plan[/THINK]",
550        ] {
551            assert_eq!(
552                tokenizer.encode(text).unwrap().token_ids(),
553                tokenizer.bpe.encode_ordinary(text)
554            );
555            assert_eq!(
556                tokenizer
557                    .encode_segments(&[EncodeSegment::control(text)])
558                    .unwrap()
559                    .token_ids(),
560                tokenizer.bpe.encode_with_special_tokens(text)
561            );
562        }
563
564        let long = "A".repeat(100_000);
565        assert_eq!(
566            tokenizer.encode(&long).unwrap().token_ids(),
567            tokenizer.bpe.encode_ordinary(&long)
568        );
569    }
570
571    #[test]
572    fn long_regex_failures_are_returned_from_all_encoding_entry_points() {
573        let tokenizer = fixture();
574        for text in ["A".repeat(1_000_000), " ".repeat(1_000_000)] {
575            let error = tokenizer.encode(&text).unwrap_err();
576            assert!(error.to_string().contains("split regex"), "{error:#}");
577            assert!(tokenizer.encode_batch(&[&text]).is_err());
578            assert!(
579                tokenizer
580                    .encode_segments(&[EncodeSegment::ordinary(&text)])
581                    .is_err()
582            );
583            let controlled = format!("<s>[INST]{text}[/INST]");
584            assert!(
585                tokenizer
586                    .encode_segments(&[EncodeSegment::control(&controlled)])
587                    .is_err()
588            );
589            assert!(tokenizer.token_to_id(&text).is_err());
590        }
591    }
592
593    /// Load the fixture after `edit`, and return the load error.
594    fn load_error(edit: impl FnOnce(&mut serde_json::Value)) -> String {
595        let mut file: serde_json::Value =
596            serde_json::from_str(&std::fs::read_to_string(FIXTURE).unwrap()).unwrap();
597        edit(&mut file);
598        let file: TekkenFile = serde_json::from_value(file).unwrap();
599        TekkenTokenizer::from_parts(file).err().unwrap().to_string()
600    }
601
602    #[test]
603    fn rejects_empty_special_tokens() {
604        let error = load_error(|file| file["special_tokens"][5]["token_str"] = "".into());
605        assert!(error.contains("empty"), "{error}");
606    }
607
608    #[test]
609    fn rejects_vocab_without_the_byte_alphabet() {
610        let error = load_error(|file| {
611            let bytes = base64::engine::general_purpose::STANDARD.encode([0xff, 0xfe, 0xfd]);
612            file["vocab"][65]["token_bytes"] = bytes.into();
613        });
614        assert!(error.contains("single byte 65"), "{error}");
615
616        let error = load_error(|file| file["config"]["default_vocab_size"] = (40 + 255).into());
617        assert!(error.contains("fewer than the 256"), "{error}");
618    }
619
620    #[test]
621    fn rejects_oversized_special_token_reservation() {
622        let error = load_error(|file| {
623            file["config"]["default_vocab_size"] = u32::MAX.into();
624            file["config"]["default_num_special_tokens"] = u32::MAX.into();
625            file["special_tokens"] = serde_json::json!([]);
626            file["vocab"] = serde_json::json!([]);
627        });
628        assert!(error.contains("special tokens, more than"), "{error}");
629    }
630
631    #[test]
632    fn prefix_cache_is_refused() {
633        assert!(fixture().validate_prefix_cache().is_err());
634    }
635
636    #[test]
637    fn recognizes_tekken_file_names() {
638        assert!(TekkenTokenizer::is_tekken_file(Path::new("/m/tekken.json")));
639        assert!(TekkenTokenizer::is_tekken_file(Path::new(
640            "tekken_240911.json"
641        )));
642        assert!(!TekkenTokenizer::is_tekken_file(Path::new(
643            "/m/tokenizer.json"
644        )));
645        assert!(!TekkenTokenizer::is_tekken_file(Path::new(
646            "/m/tekken.model"
647        )));
648    }
649}