Skip to main content

dynamo_tokenizers/
hf.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{collections::HashSet, path::Path, sync::OnceLock};
5
6use tokenizers::tokenizer::{
7    AddedToken, AddedVocabulary, PostProcessor as _, Tokenizer as HfTokenizer,
8};
9
10use super::{
11    Encoding, Error, Result, TokenIdType, TokenizerOptions,
12    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
13};
14
15pub struct HuggingFaceTokenizer {
16    tokenizer: HfTokenizer,
17    segment_tokenizers: OnceLock<(HfTokenizer, HfTokenizer)>,
18    byte_fallback_ids: HashSet<u32>,
19    special_ids: HashSet<u32>,
20    /// Every added-token id, `special: true` or not. An ordinary segment of
21    /// [`Encoder::encode_segments`] must never produce one of these; see
22    /// that method.
23    added_ids: HashSet<u32>,
24    /// Construction-time options; see [`TokenizerOptions`]. Defaults preserve
25    /// the historical behavior (e.g. `add_special_tokens: false`). Set via
26    /// [`Tokenizer::with_options`].
27    options: TokenizerOptions,
28}
29
30impl HuggingFaceTokenizer {
31    /// Every added-token id of this vocabulary, `special: true` or not.
32    pub(crate) fn added_ids(&self) -> &HashSet<u32> {
33        &self.added_ids
34    }
35
36    /// Load from `tokenizer.json`, merging in special tokens declared only in
37    /// a sibling `tokenizer_config.json`'s `added_tokens_decoder`. Without
38    /// this, some releases (Qwen2-VL-2B's `<|image_pad|>`) BPE-shatter and
39    /// silently break MM-aware routing. The merge is idempotent.
40    pub fn from_file(model_name: &str) -> Result<Self> {
41        let mut tokenizer = HfTokenizer::from_file(model_name)
42            .map_err(|err| Error::msg(format!("Error loading tokenizer: {}", err)))?;
43
44        if let Some(parent) = Path::new(model_name).parent() {
45            merge_special_tokens_from_config(&mut tokenizer, parent);
46        }
47
48        Ok(Self::from_tokenizer(tokenizer))
49    }
50
51    pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self {
52        // ByteFallback decodes a contiguous run atomically: one invalid byte
53        // replaces the entire run, including previously valid ASCII/UTF-8.
54        // Discover the at-most-256 byte IDs once, not on the request hot path.
55        let has_byte_fallback = tokenizer.get_decoder().is_some_and(|decoder| {
56            fn contains(value: &serde_json::Value) -> bool {
57                value["type"] == "ByteFallback"
58                    || value["decoders"]
59                        .as_array()
60                        .is_some_and(|items| items.iter().any(contains))
61            }
62            serde_json::to_value(decoder).is_ok_and(|value| contains(&value))
63        });
64        let byte_fallback_ids = if has_byte_fallback {
65            tokenizer
66                .get_vocab(true)
67                .into_iter()
68                .filter_map(|(token, id)| {
69                    (token.len() == 6
70                        && token.starts_with("<0x")
71                        && token.ends_with('>')
72                        && u8::from_str_radix(&token[3..5], 16).is_ok())
73                    .then_some(id)
74                })
75                .collect()
76        } else {
77            HashSet::new()
78        };
79        let added_tokens = tokenizer.get_added_tokens_decoder();
80        let special_ids = added_tokens
81            .iter()
82            .filter_map(|(id, token)| token.special.then_some(*id))
83            .collect();
84        let added_ids = added_tokens.into_keys().collect();
85        HuggingFaceTokenizer {
86            tokenizer,
87            segment_tokenizers: OnceLock::new(),
88            byte_fallback_ids,
89            special_ids,
90            added_ids,
91            options: TokenizerOptions::default(),
92        }
93    }
94
95    /// Wrap an already-loaded `HfTokenizer` and merge in the sibling
96    /// `tokenizer_config.json` special tokens; see [`Self::from_file`].
97    pub fn from_tokenizer_with_model_dir(tokenizer: HfTokenizer, model_dir: &Path) -> Self {
98        let mut tokenizer = tokenizer;
99        merge_special_tokens_from_config(&mut tokenizer, model_dir);
100        Self::from_tokenizer(tokenizer)
101    }
102}
103
104/// Promote `tokenizer_config.json`'s `special: true` `added_tokens_decoder`
105/// entries onto `tokenizer`. Missing-file / parse errors are swallowed since
106/// the file is optional. See [`HuggingFaceTokenizer::from_file`].
107///
108/// `pub` so downstream crates (e.g. `dynamo-llm`'s model_card) can apply
109/// the same promotion before extracting the special-token boundary list
110/// for the L1 prefix cache — otherwise the cache and the wrapped
111/// tokenizer would disagree on which strings are atomic specials.
112pub fn merge_special_tokens_from_config(tokenizer: &mut HfTokenizer, model_dir: &Path) {
113    let cfg_path = model_dir.join("tokenizer_config.json");
114    let Ok(raw) = std::fs::read_to_string(&cfg_path) else {
115        return;
116    };
117    let cfg: serde_json::Value = match serde_json::from_str(&raw) {
118        Ok(v) => v,
119        Err(e) => {
120            tracing::debug!(
121                target: "tokenizer",
122                path = %cfg_path.display(),
123                error = %e,
124                "tokenizer_config.json parse failed; skipping special-token merge"
125            );
126            return;
127        }
128    };
129    let Some(decoder) = cfg.get("added_tokens_decoder").and_then(|v| v.as_object()) else {
130        return;
131    };
132
133    let mut to_add: Vec<AddedToken> = Vec::new();
134    for (_id, spec) in decoder {
135        let obj = match spec.as_object() {
136            Some(o) => o,
137            None => continue,
138        };
139        // The id is informational — `add_special_tokens` reuses the existing
140        // vocab id via `Model::token_to_id` on the content string.
141        if obj.get("special").and_then(|v| v.as_bool()) != Some(true) {
142            continue;
143        }
144        let Some(content) = obj.get("content").and_then(|v| v.as_str()) else {
145            continue;
146        };
147        if content.is_empty() {
148            continue;
149        }
150        let single_word = obj
151            .get("single_word")
152            .and_then(|v| v.as_bool())
153            .unwrap_or(false);
154        let lstrip = obj.get("lstrip").and_then(|v| v.as_bool()).unwrap_or(false);
155        let rstrip = obj.get("rstrip").and_then(|v| v.as_bool()).unwrap_or(false);
156        let normalized = obj
157            .get("normalized")
158            .and_then(|v| v.as_bool())
159            .unwrap_or(false);
160        let token = AddedToken::from(content.to_string(), true)
161            .single_word(single_word)
162            .lstrip(lstrip)
163            .rstrip(rstrip)
164            .normalized(normalized);
165        to_add.push(token);
166    }
167
168    if to_add.is_empty() {
169        return;
170    }
171    // Dedups against existing added-tokens, so this is a no-op when
172    // tokenizer.json already had them. Return value = net-new count.
173    let added = match tokenizer.add_special_tokens(to_add.iter().cloned()) {
174        Ok(added) => added,
175        Err(e) => {
176            tracing::warn!(
177                target: "tokenizer",
178                path = %cfg_path.display(),
179                error = %e,
180                "special-token merge from tokenizer_config.json failed; skipping"
181            );
182            return;
183        }
184    };
185    if added > 0 {
186        // Warn (not debug) when the merge actually promotes anything —
187        // intentionally loud so accidental promotion of a previously-
188        // non-special token shows up immediately in worker logs. Lists
189        // the literal token strings so debugging doesn't require a
190        // second pass through `added_tokens_decoder`.
191        let promoted: Vec<&str> = to_add.iter().map(|t| t.content.as_str()).collect();
192        tracing::warn!(
193            target: "tokenizer",
194            path = %cfg_path.display(),
195            added,
196            candidates = to_add.len(),
197            promoted = ?promoted,
198            "merged additional special tokens from tokenizer_config.json"
199        );
200    }
201}
202
203impl Encoder for HuggingFaceTokenizer {
204    fn encode(&self, input: &str) -> Result<Encoding> {
205        // This self.tokenizer is the library
206        let encoding = self
207            .tokenizer
208            .encode(input, self.options.add_special_tokens)
209            .map_err(|err| Error::msg(format!("Error tokenizing input: {err}")))?;
210
211        Ok(Encoding::Hf(Box::new(encoding)))
212    }
213
214    fn encode_segments(&self, segments: &[super::EncodeSegment<'_>]) -> Result<Encoding> {
215        let (trusted, ordinary) = self.segment_tokenizers.get_or_init(|| {
216            let mut trusted = self.tokenizer.clone();
217            // Apply padding, truncation and post-processing once to the joined
218            // sequence, rather than independently at every trust boundary.
219            trusted.with_padding(None);
220            trusted
221                .with_truncation(None)
222                .expect("removing truncation cannot fail");
223            trusted.with_post_processor(None::<tokenizers::processors::PostProcessorWrapper>);
224            // No added vocabulary at all: `encode_special_tokens` skips only
225            // `special: true` entries, and chat markers such as Qwen3's
226            // `<think>` are `special: false`.
227            let mut ordinary = trusted.clone();
228            ordinary.with_added_vocabulary(AddedVocabulary::new());
229            (trusted, ordinary)
230        });
231        let encodings = segments
232            .iter()
233            .map(|segment| {
234                let tokenizer = if segment.allow_special {
235                    trusted
236                } else {
237                    ordinary
238                };
239                let encoding = tokenizer
240                    .encode(segment.text, false)
241                    .map_err(|error| Error::msg(format!("Error tokenizing segment: {error}")))?;
242                if !segment.allow_special {
243                    crate::reject_control_ids_in_ordinary_segment(
244                        encoding.get_ids(),
245                        &self.added_ids,
246                    )?;
247                }
248                Ok(encoding)
249            })
250            .collect::<Result<Vec<_>>>()?;
251        let merged = tokenizers::Encoding::merge(encodings, true);
252        let encoding = self
253            .tokenizer
254            .post_process(merged, None, self.options.add_special_tokens)
255            .map_err(|error| Error::msg(format!("Error post-processing segments: {error}")))?;
256        Ok(Encoding::Hf(Box::new(encoding)))
257    }
258
259    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
260        let hf_encodings = self
261            .tokenizer
262            .encode_batch(inputs.to_vec(), self.options.add_special_tokens)
263            .map_err(|err| Error::msg(format!("Error batch tokenizing input: {err}")))?;
264
265        let encodings = hf_encodings
266            .into_iter()
267            .map(|enc| Encoding::Hf(Box::new(enc)))
268            .collect();
269
270        Ok(encodings)
271    }
272}
273
274impl Decoder for HuggingFaceTokenizer {
275    fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
276        token_ids
277            .iter()
278            .rev()
279            .find(|id| !skip_special_tokens || !self.special_ids.contains(id))
280            .is_some_and(|id| self.byte_fallback_ids.contains(id))
281    }
282
283    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
284        // This calls into the library
285        let text = self
286            .tokenizer
287            .decode(token_ids, skip_special_tokens)
288            .map_err(|err| Error::msg(format!("Error de-tokenizing input: {err}")))?;
289
290        Ok(text.into())
291    }
292}
293
294impl Tokenizer for HuggingFaceTokenizer {
295    fn validate_prefix_cache(&self) -> Result<()> {
296        if self.options.add_special_tokens {
297            return Err(Error::msg(
298                "HuggingFace tokenizers configured with add_special_tokens=true must remain uncached",
299            ));
300        }
301        // The cache joins separately encoded pieces, so per-call truncation or padding
302        // would apply to each piece instead of the whole input.
303        if self.tokenizer.get_truncation().is_some() || self.tokenizer.get_padding().is_some() {
304            return Err(Error::msg(
305                "HuggingFace tokenizers with truncation or padding must remain uncached",
306            ));
307        }
308        Ok(())
309    }
310
311    /// Without added special tokens, truncation or padding, a segment that allows
312    /// special tokens goes through the pipeline `encode` runs.
313    fn validate_segment_prefix_cache(&self) -> Result<()> {
314        self.validate_prefix_cache()
315    }
316
317    /// HF honors [`TokenizerOptions::add_special_tokens`] (BOS/EOS per the
318    /// tokenizer's post-processor template).
319    fn with_options(mut self, options: TokenizerOptions) -> Self {
320        self.options = options;
321        self
322    }
323
324    fn vocab_size(&self) -> Option<usize> {
325        Some(self.tokenizer.get_vocab_size(true))
326    }
327
328    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
329        Ok(self.tokenizer.token_to_id(token))
330    }
331
332    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
333        let mut ids: Vec<TokenIdType> = self
334            .tokenizer
335            .get_added_tokens_decoder()
336            .into_iter()
337            .filter_map(|(id, token)| token.special.then_some(id))
338            .collect();
339        ids.sort_unstable();
340        Ok(ids)
341    }
342
343    fn num_special_tokens_added(&self) -> Result<usize> {
344        Ok(self
345            .tokenizer
346            .get_post_processor()
347            .map_or(0, |processor| processor.added_tokens(false)))
348    }
349}
350
351impl From<HfTokenizer> for HuggingFaceTokenizer {
352    fn from(tokenizer: HfTokenizer) -> Self {
353        Self::from_tokenizer(tokenizer)
354    }
355}
356
357#[cfg(test)]
358mod byte_fallback_stream_tests {
359    use super::*;
360
361    fn tokenizer() -> super::super::Tokenizer {
362        let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
363            "version": "1.0", "truncation": null, "padding": null,
364            "added_tokens": [{"id": 6, "content": "<eos>", "special": true,
365                "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}],
366            "normalizer": null, "pre_tokenizer": null, "post_processor": null,
367            "decoder": {"type": "Sequence", "decoders": [
368                {"type": "ByteFallback"}, {"type": "Fuse"}]},
369            "model": {"type": "BPE", "vocab": {
370                "<0x61>": 0, "<0xF5>": 1, "<0xC3>": 2, "<0xA9>": 3,
371                " hello": 4, "!": 5, "<eos>": 6}, "merges": [], "byte_fallback": true}
372        }))
373        .unwrap();
374        std::sync::Arc::new(HuggingFaceTokenizer::from_tokenizer(hf)).into()
375    }
376
377    #[test]
378    fn byte_runs_match_full_decode() {
379        let tokenizer = tokenizer();
380        // Invalid tails rewrite the whole byte run, even previously valid ASCII
381        // or a complete multibyte character. Valid, incomplete and literal
382        // replacement output must all survive a terminal flush unchanged.
383        for (ids, skip) in [
384            (vec![0, 1, 4], false),
385            (vec![2, 3, 1, 4], false),
386            (vec![2, 3, 5], false),
387            (vec![0], false),
388            (vec![2, 3], false),
389            (vec![2], false),
390            (vec![0, 1], false),
391            (vec![0, 6, 1, 4], false),
392            (vec![0, 6, 1, 4], true),
393        ] {
394            let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
395            let mut stream = tokenizer.decode_stream(&[], skip);
396            let mut actual = String::new();
397            for id in &ids {
398                actual.push_str(&stream.step(*id).unwrap().unwrap_or_default());
399            }
400            actual.push_str(&stream.finish().unwrap().unwrap_or_default());
401            assert_eq!(actual, expected, "ids={ids:?}, skip={skip}");
402            assert_eq!(stream.finish().unwrap(), None);
403        }
404    }
405
406    #[test]
407    fn only_byte_suffix_is_delayed_and_prompt_is_not_emitted() {
408        let tokenizer = tokenizer();
409        let mut stream = tokenizer.decode_stream(&[4], true);
410        assert_eq!(stream.step(5).unwrap().as_deref(), Some("!"));
411        assert_eq!(stream.step(0).unwrap(), None);
412        assert_eq!(stream.step(1).unwrap(), None);
413        assert_eq!(stream.step(4).unwrap().as_deref(), Some("�� hello"));
414        assert_eq!(stream.finish().unwrap(), None);
415    }
416}
417
418#[cfg(test)]
419mod tests {
420    //! The existing tool-calling / reasoning parser tests inject already-
421    //! decoded text and never run `tokenizer.decode`, so they cannot
422    //! catch the class of bug where a non-special marker gets accidentally
423    //! promoted to "special" and silently disappears under
424    //! `skip_special_tokens=True`. One unit test pins both halves of the
425    //! gate (promote special:true / skip special:false) end-to-end through
426    //! the actual encode -> decode round trip — the layer above which the
427    //! parser tests cannot reach.
428    use super::*;
429    use std::fs;
430    use tempfile::TempDir;
431
432    #[test]
433    fn merge_gate_round_trips_through_decode() {
434        // Minimal WordLevel `tokenizer.json`. `<|special_kept|>` and
435        // `<|special_dropped|>` are deliberately NOT pre-declared as
436        // special in `added_tokens` here — that's exactly the shape
437        // `merge_special_tokens_from_config` is supposed to fix from
438        // tokenizer_config.json.
439        const TOKENIZER_JSON: &str = r#"{
440            "version": "1.0",
441            "truncation": null,
442            "padding": null,
443            "added_tokens": [
444                {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
445            ],
446            "normalizer": null,
447            "pre_tokenizer": null,
448            "post_processor": null,
449            "decoder": null,
450            "model": {
451                "type": "WordLevel",
452                "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<|special_kept|>": 3, "<|special_dropped|>": 4},
453                "unk_token": "<unk>"
454            }
455        }"#;
456
457        // `<|special_kept|>` is `special: true` → must be promoted, and
458        // therefore stripped under skip_special_tokens=true.
459        // `<|special_dropped|>` is `special: false` → must be skipped,
460        // and therefore survive skip_special_tokens=true. The latter is
461        // the Ryan/Keiven concern: tool-call / reasoning markers are
462        // universally declared `special: false` precisely so the parser
463        // still sees them; a regression here would silently break
464        // parsing without any of the parser unit tests turning red.
465        const TOKENIZER_CONFIG_JSON: &str = r#"{
466            "added_tokens_decoder": {
467                "3": {"content": "<|special_kept|>",    "special": true,  "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
468                "4": {"content": "<|special_dropped|>", "special": false, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
469            }
470        }"#;
471
472        let dir = TempDir::new().unwrap();
473        fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
474        fs::write(
475            dir.path().join("tokenizer_config.json"),
476            TOKENIZER_CONFIG_JSON,
477        )
478        .unwrap();
479
480        let mut tokenizer = HfTokenizer::from_file(dir.path().join("tokenizer.json")).unwrap();
481        merge_special_tokens_from_config(&mut tokenizer, dir.path());
482
483        // Registry assertion: only the special:true entry was promoted.
484        let specials: Vec<String> = {
485            let mut v: Vec<String> = tokenizer
486                .get_added_tokens_decoder()
487                .values()
488                .filter(|t| t.special)
489                .map(|t| t.content.clone())
490                .collect();
491            v.sort();
492            v
493        };
494        assert_eq!(
495            specials,
496            vec!["<unk>".to_string(), "<|special_kept|>".to_string()],
497            "<|special_kept|> promoted; <|special_dropped|> stayed non-special"
498        );
499
500        // Decode round-trip assertion (the layer parser tests cannot
501        // reach): the registry mutation actually changes downstream
502        // decode behavior in the expected direction.
503        let enc_kept = tokenizer.encode("<|special_kept|>", false).unwrap();
504        let decoded_strip = tokenizer.decode(enc_kept.get_ids(), true).unwrap();
505        assert!(
506            !decoded_strip.contains("<|special_kept|>"),
507            "promoted special:true token must be stripped under skip_special_tokens=true; got {decoded_strip:?}"
508        );
509
510        let enc_drop = tokenizer.encode("<|special_dropped|>", false).unwrap();
511        let decoded_keep = tokenizer.decode(enc_drop.get_ids(), true).unwrap();
512        assert!(
513            decoded_keep.contains("<|special_dropped|>"),
514            "non-promoted special:false token must survive skip_special_tokens=true; got {decoded_keep:?}"
515        );
516    }
517
518    #[test]
519    fn add_special_tokens_flag_controls_encode() {
520        // WordLevel tokenizer with a TemplateProcessing post-processor that
521        // prepends <bos>. The post-processor only fires when encode is
522        // called with add_special_tokens=true, which is exactly the knob
523        // `TokenizerOptions`/`with_options` exposes.
524        const TOKENIZER_JSON: &str = r#"{
525            "version": "1.0",
526            "truncation": null,
527            "padding": null,
528            "added_tokens": [
529                {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
530                {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
531            ],
532            "normalizer": null,
533            "pre_tokenizer": null,
534            "post_processor": {
535                "type": "TemplateProcessing",
536                "single": [
537                    {"SpecialToken": {"id": "<bos>", "type_id": 0}},
538                    {"Sequence": {"id": "A", "type_id": 0}}
539                ],
540                "pair": [
541                    {"SpecialToken": {"id": "<bos>", "type_id": 0}},
542                    {"Sequence": {"id": "A", "type_id": 0}},
543                    {"Sequence": {"id": "B", "type_id": 0}}
544                ],
545                "special_tokens": {
546                    "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
547                }
548            },
549            "decoder": null,
550            "model": {
551                "type": "WordLevel",
552                "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
553                "unk_token": "<unk>"
554            }
555        }"#;
556
557        let dir = TempDir::new().unwrap();
558        fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
559        let path = dir.path().join("tokenizer.json");
560        let path = path.to_str().unwrap();
561
562        let ids = |enc: &Encoding| match enc {
563            Encoding::Hf(e) => e.get_ids().to_vec(),
564            _ => panic!("expected Hf encoding"),
565        };
566
567        // Default: unchanged historical behavior — no BOS.
568        let plain = HuggingFaceTokenizer::from_file(path).unwrap();
569        assert_eq!(ids(&plain.encode("hello").unwrap()), vec![1]);
570
571        let with_bos =
572            HuggingFaceTokenizer::from_file(path)
573                .unwrap()
574                .with_options(TokenizerOptions {
575                    add_special_tokens: true,
576                });
577        assert_eq!(ids(&with_bos.encode("hello").unwrap()), vec![3, 1]);
578        let batch = with_bos.encode_batch(&["hello", "world"]).unwrap();
579        assert_eq!(ids(&batch[0]), vec![3, 1]);
580        assert_eq!(ids(&batch[1]), vec![3, 2]);
581
582        // Same knob through the top-level wrapper: from_file keeps the
583        // historical default, from_file_with_options threads the flag to
584        // construction before the tokenizer goes behind the shared Arc.
585        use crate::Tokenizer as TokenizerWrapper;
586        let wrapper_plain = TokenizerWrapper::from_file(path).unwrap();
587        assert_eq!(ids(&wrapper_plain.encode("hello").unwrap()), vec![1]);
588
589        let wrapper_bos = TokenizerWrapper::from_file_with_options(
590            path,
591            TokenizerOptions {
592                add_special_tokens: true,
593            },
594        )
595        .unwrap();
596        assert_eq!(ids(&wrapper_bos.encode("hello").unwrap()), vec![3, 1]);
597    }
598
599    #[test]
600    fn truncation_or_padding_blocks_prefix_cache() {
601        let path = concat!(
602            env!("CARGO_MANIFEST_DIR"),
603            "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
604        );
605        let mut truncated = HfTokenizer::from_file(path).unwrap();
606        truncated
607            .with_truncation(Some(tokenizers::TruncationParams::default()))
608            .unwrap();
609        let mut padded = HfTokenizer::from_file(path).unwrap();
610        padded.with_padding(Some(tokenizers::PaddingParams::default()));
611        for hf in [truncated, padded] {
612            assert!(
613                HuggingFaceTokenizer::from_tokenizer(hf)
614                    .validate_prefix_cache()
615                    .is_err()
616            );
617        }
618    }
619
620    #[test]
621    fn vocab_introspection_accessors() {
622        const TOKENIZER_JSON: &str = r#"{
623            "version": "1.0",
624            "truncation": null,
625            "padding": null,
626            "added_tokens": [
627                {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
628            ],
629            "normalizer": null,
630            "pre_tokenizer": null,
631            "post_processor": null,
632            "decoder": null,
633            "model": {
634                "type": "WordLevel",
635                "vocab": {"<unk>": 0, "hello": 1, "world": 2},
636                "unk_token": "<unk>"
637            }
638        }"#;
639
640        let dir = TempDir::new().unwrap();
641        fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
642        let path = dir.path().join("tokenizer.json");
643
644        let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
645        assert_eq!(tokenizer.vocab_size(), Some(3));
646        assert_eq!(tokenizer.token_to_id("hello").unwrap(), Some(1));
647        assert_eq!(tokenizer.special_token_ids().unwrap(), vec![0]);
648        assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 0);
649    }
650
651    #[test]
652    fn num_special_tokens_added_reflects_post_processor_additions() {
653        const TOKENIZER_JSON: &str = r#"{
654            "version": "1.0",
655            "truncation": null,
656            "padding": null,
657            "added_tokens": [
658                {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
659                {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
660            ],
661            "normalizer": null,
662            "pre_tokenizer": null,
663            "post_processor": {
664                "type": "TemplateProcessing",
665                "single": [
666                    {"SpecialToken": {"id": "<bos>", "type_id": 0}},
667                    {"Sequence": {"id": "A", "type_id": 0}}
668                ],
669                "pair": [
670                    {"SpecialToken": {"id": "<bos>", "type_id": 0}},
671                    {"Sequence": {"id": "A", "type_id": 0}},
672                    {"Sequence": {"id": "B", "type_id": 0}}
673                ],
674                "special_tokens": {
675                    "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
676                }
677            },
678            "decoder": null,
679            "model": {
680                "type": "WordLevel",
681                "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
682                "unk_token": "<unk>"
683            }
684        }"#;
685
686        let dir = TempDir::new().unwrap();
687        fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
688        let path = dir.path().join("tokenizer.json");
689
690        let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
691        assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 1);
692    }
693}
694
695#[cfg(test)]
696mod segmented_tests {
697    use super::*;
698    use crate::EncodeSegment;
699
700    #[test]
701    fn segmented_hf_preserves_literal_controls_and_adds_bos_once() {
702        let mut vocab = serde_json::json!({"<unk>":0,"<|call|>":1,"<bos>":2});
703        for ch in "<|cal>".chars() {
704            let id = vocab.as_object().unwrap().len();
705            vocab[ch.to_string()] = serde_json::json!(id);
706        }
707        let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
708            "version":"1.0", "truncation":null, "padding":null,
709            "normalizer":null, "pre_tokenizer":null, "decoder":{"type":"Fuse"},
710            "added_tokens":[{"id":1,"content":"<|call|>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false},
711                {"id":2,"content":"<bos>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
712            "post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<bos>","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],
713                "pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"<bos>":{"id":"<bos>","ids":[2],"tokens":["<bos>"]}}},
714            "model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
715        })).unwrap();
716        for add_special_tokens in [false, true] {
717            let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf.clone())
718                .with_options(TokenizerOptions { add_special_tokens });
719            let encoding = tokenizer
720                .encode_segments(&[
721                    EncodeSegment::control("<|call|>"),
722                    EncodeSegment::ordinary("<|call|>"),
723                ])
724                .unwrap();
725            let ids = encoding.token_ids();
726            assert_eq!(ids.iter().filter(|id| **id == 1).count(), 1);
727            assert_eq!(
728                ids.iter().filter(|id| **id == 2).count(),
729                usize::from(add_special_tokens)
730            );
731            let decoded: String = tokenizer.decode(ids, true).unwrap().into();
732            assert_eq!(decoded, "<|call|>");
733            assert_eq!(
734                tokenizer
735                    .encode("<|call|>")
736                    .unwrap()
737                    .token_ids()
738                    .iter()
739                    .filter(|id| **id == 1)
740                    .count(),
741                1
742            );
743        }
744    }
745    #[test]
746    fn segmented_hf_encodes_non_special_added_tokens_as_text() {
747        // HF's `set_encode_special_tokens` skips only `special: true` entries;
748        // chat markers such as Qwen3's `<think>` are registered `special: false`.
749        let mut vocab = serde_json::json!({"<unk>":0,"<think>":1});
750        for ch in "<think>".chars() {
751            let id = vocab.as_object().unwrap().len();
752            vocab[ch.to_string()] = serde_json::json!(id);
753        }
754        let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
755            "version":"1.0", "truncation":null, "padding":null,
756            "normalizer":null, "pre_tokenizer":null, "post_processor":null, "decoder":{"type":"Fuse"},
757            "added_tokens":[{"id":1,"content":"<think>","special":false,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
758            "model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
759        })).unwrap();
760        let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf);
761        let ids = |segment| {
762            tokenizer
763                .encode_segments(&[segment])
764                .unwrap()
765                .token_ids()
766                .to_vec()
767        };
768        assert_eq!(ids(EncodeSegment::control("<think>")), [1]);
769        let ordinary = ids(EncodeSegment::ordinary("<think>"));
770        assert_eq!(ordinary.len(), "<think>".len());
771        assert!(!ordinary.contains(&1));
772    }
773
774    #[test]
775    fn segmented_hf_rejects_ordinary_control_ids() {
776        // `H`+`e` merges to id 18, declared here as the added token `He`:
777        // ordinary text must not come back as that control id.
778        let mut json: serde_json::Value = serde_json::from_str(
779            &std::fs::read_to_string(concat!(
780                env!("CARGO_MANIFEST_DIR"),
781                "/tests/data/minimal-bpe/tokenizer.json"
782            ))
783            .unwrap(),
784        )
785        .unwrap();
786        json["added_tokens"] = serde_json::json!([{
787            "id": 18,
788            "content": "He",
789            "single_word": false,
790            "lstrip": false,
791            "rstrip": false,
792            "normalized": false,
793            "special": true
794        }]);
795        let tokenizer = HuggingFaceTokenizer::from_tokenizer(serde_json::from_value(json).unwrap());
796        let control = tokenizer
797            .encode_segments(&[EncodeSegment::control("He")])
798            .unwrap();
799        assert_eq!(control.token_ids(), &[18]);
800        let error = tokenizer
801            .encode_segments(&[EncodeSegment::ordinary("He")])
802            .unwrap_err();
803        assert!(
804            error.to_string().contains("control-token id 18"),
805            "unexpected error: {error}"
806        );
807    }
808}