dynamo_tokenizers/
hf.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::path::Path;
5
6use tokenizers::tokenizer::{AddedToken, Tokenizer as HfTokenizer};
7
8use super::{
9    Encoding, Error, Result, TokenIdType,
10    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
11};
12
13pub struct HuggingFaceTokenizer {
14    tokenizer: HfTokenizer,
15}
16
17impl HuggingFaceTokenizer {
18    /// Load from `tokenizer.json`, merging in special tokens declared only in
19    /// a sibling `tokenizer_config.json`'s `added_tokens_decoder`. Without
20    /// this, some releases (Qwen2-VL-2B's `<|image_pad|>`) BPE-shatter and
21    /// silently break MM-aware routing. The merge is idempotent.
22    pub fn from_file(model_name: &str) -> Result<Self> {
23        let mut tokenizer = HfTokenizer::from_file(model_name)
24            .map_err(|err| Error::msg(format!("Error loading tokenizer: {}", err)))?;
25
26        if let Some(parent) = Path::new(model_name).parent() {
27            merge_special_tokens_from_config(&mut tokenizer, parent);
28        }
29
30        Ok(HuggingFaceTokenizer { tokenizer })
31    }
32
33    pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self {
34        HuggingFaceTokenizer { tokenizer }
35    }
36
37    /// Wrap an already-loaded `HfTokenizer` and merge in the sibling
38    /// `tokenizer_config.json` special tokens; see [`Self::from_file`].
39    pub fn from_tokenizer_with_model_dir(tokenizer: HfTokenizer, model_dir: &Path) -> Self {
40        let mut tokenizer = tokenizer;
41        merge_special_tokens_from_config(&mut tokenizer, model_dir);
42        HuggingFaceTokenizer { tokenizer }
43    }
44}
45
46/// Promote `tokenizer_config.json`'s `special: true` `added_tokens_decoder`
47/// entries onto `tokenizer`. Missing-file / parse errors are swallowed since
48/// the file is optional. See [`HuggingFaceTokenizer::from_file`].
49///
50/// `pub` so downstream crates (e.g. `dynamo-llm`'s model_card) can apply
51/// the same promotion before extracting the special-token boundary list
52/// for the L1 prefix cache — otherwise the cache and the wrapped
53/// tokenizer would disagree on which strings are atomic specials.
54pub fn merge_special_tokens_from_config(tokenizer: &mut HfTokenizer, model_dir: &Path) {
55    let cfg_path = model_dir.join("tokenizer_config.json");
56    let Ok(raw) = std::fs::read_to_string(&cfg_path) else {
57        return;
58    };
59    let cfg: serde_json::Value = match serde_json::from_str(&raw) {
60        Ok(v) => v,
61        Err(e) => {
62            tracing::debug!(
63                target: "tokenizer",
64                path = %cfg_path.display(),
65                error = %e,
66                "tokenizer_config.json parse failed; skipping special-token merge"
67            );
68            return;
69        }
70    };
71    let Some(decoder) = cfg.get("added_tokens_decoder").and_then(|v| v.as_object()) else {
72        return;
73    };
74
75    let mut to_add: Vec<AddedToken> = Vec::new();
76    for (_id, spec) in decoder {
77        let obj = match spec.as_object() {
78            Some(o) => o,
79            None => continue,
80        };
81        // The id is informational — `add_special_tokens` reuses the existing
82        // vocab id via `Model::token_to_id` on the content string.
83        if obj.get("special").and_then(|v| v.as_bool()) != Some(true) {
84            continue;
85        }
86        let Some(content) = obj.get("content").and_then(|v| v.as_str()) else {
87            continue;
88        };
89        if content.is_empty() {
90            continue;
91        }
92        let single_word = obj
93            .get("single_word")
94            .and_then(|v| v.as_bool())
95            .unwrap_or(false);
96        let lstrip = obj.get("lstrip").and_then(|v| v.as_bool()).unwrap_or(false);
97        let rstrip = obj.get("rstrip").and_then(|v| v.as_bool()).unwrap_or(false);
98        let normalized = obj
99            .get("normalized")
100            .and_then(|v| v.as_bool())
101            .unwrap_or(false);
102        let token = AddedToken::from(content.to_string(), true)
103            .single_word(single_word)
104            .lstrip(lstrip)
105            .rstrip(rstrip)
106            .normalized(normalized);
107        to_add.push(token);
108    }
109
110    if to_add.is_empty() {
111        return;
112    }
113    // Dedups against existing added-tokens, so this is a no-op when
114    // tokenizer.json already had them. Return value = net-new count.
115    let added = tokenizer.add_special_tokens(&to_add);
116    if added > 0 {
117        // Warn (not debug) when the merge actually promotes anything —
118        // intentionally loud so accidental promotion of a previously-
119        // non-special token shows up immediately in worker logs. Lists
120        // the literal token strings so debugging doesn't require a
121        // second pass through `added_tokens_decoder`.
122        let promoted: Vec<&str> = to_add.iter().map(|t| t.content.as_str()).collect();
123        tracing::warn!(
124            target: "tokenizer",
125            path = %cfg_path.display(),
126            added,
127            candidates = to_add.len(),
128            promoted = ?promoted,
129            "merged additional special tokens from tokenizer_config.json"
130        );
131    }
132}
133
134impl Encoder for HuggingFaceTokenizer {
135    fn encode(&self, input: &str) -> Result<Encoding> {
136        // This self.tokenizer is the library
137        let encoding = self
138            .tokenizer
139            .encode(input, false)
140            .map_err(|err| Error::msg(format!("Error tokenizing input: {err}")))?;
141
142        Ok(Encoding::Hf(Box::new(encoding)))
143    }
144
145    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
146        let hf_encodings = self
147            .tokenizer
148            .encode_batch(inputs.to_vec(), false)
149            .map_err(|err| Error::msg(format!("Error batch tokenizing input: {err}")))?;
150
151        let encodings = hf_encodings
152            .into_iter()
153            .map(|enc| Encoding::Hf(Box::new(enc)))
154            .collect();
155
156        Ok(encodings)
157    }
158}
159
160impl Decoder for HuggingFaceTokenizer {
161    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
162        // This calls into the library
163        let text = self
164            .tokenizer
165            .decode(token_ids, skip_special_tokens)
166            .map_err(|err| Error::msg(format!("Error de-tokenizing input: {err}")))?;
167
168        Ok(text.into())
169    }
170}
171
172impl Tokenizer for HuggingFaceTokenizer {}
173
174impl From<HfTokenizer> for HuggingFaceTokenizer {
175    fn from(tokenizer: HfTokenizer) -> Self {
176        HuggingFaceTokenizer { tokenizer }
177    }
178}
179
180#[cfg(test)]
181mod tests {
182    //! The existing tool-calling / reasoning parser tests inject already-
183    //! decoded text and never run `tokenizer.decode`, so they cannot
184    //! catch the class of bug where a non-special marker gets accidentally
185    //! promoted to "special" and silently disappears under
186    //! `skip_special_tokens=True`. One unit test pins both halves of the
187    //! gate (promote special:true / skip special:false) end-to-end through
188    //! the actual encode -> decode round trip — the layer above which the
189    //! parser tests cannot reach.
190    use super::*;
191    use std::fs;
192    use tempfile::TempDir;
193
194    #[test]
195    fn merge_gate_round_trips_through_decode() {
196        // Minimal WordLevel `tokenizer.json`. `<|special_kept|>` and
197        // `<|special_dropped|>` are deliberately NOT pre-declared as
198        // special in `added_tokens` here — that's exactly the shape
199        // `merge_special_tokens_from_config` is supposed to fix from
200        // tokenizer_config.json.
201        const TOKENIZER_JSON: &str = r#"{
202            "version": "1.0",
203            "truncation": null,
204            "padding": null,
205            "added_tokens": [
206                {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
207            ],
208            "normalizer": null,
209            "pre_tokenizer": null,
210            "post_processor": null,
211            "decoder": null,
212            "model": {
213                "type": "WordLevel",
214                "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<|special_kept|>": 3, "<|special_dropped|>": 4},
215                "unk_token": "<unk>"
216            }
217        }"#;
218
219        // `<|special_kept|>` is `special: true` → must be promoted, and
220        // therefore stripped under skip_special_tokens=true.
221        // `<|special_dropped|>` is `special: false` → must be skipped,
222        // and therefore survive skip_special_tokens=true. The latter is
223        // the Ryan/Keiven concern: tool-call / reasoning markers are
224        // universally declared `special: false` precisely so the parser
225        // still sees them; a regression here would silently break
226        // parsing without any of the parser unit tests turning red.
227        const TOKENIZER_CONFIG_JSON: &str = r#"{
228            "added_tokens_decoder": {
229                "3": {"content": "<|special_kept|>",    "special": true,  "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
230                "4": {"content": "<|special_dropped|>", "special": false, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
231            }
232        }"#;
233
234        let dir = TempDir::new().unwrap();
235        fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
236        fs::write(
237            dir.path().join("tokenizer_config.json"),
238            TOKENIZER_CONFIG_JSON,
239        )
240        .unwrap();
241
242        let mut tokenizer = HfTokenizer::from_file(dir.path().join("tokenizer.json")).unwrap();
243        merge_special_tokens_from_config(&mut tokenizer, dir.path());
244
245        // Registry assertion: only the special:true entry was promoted.
246        let specials: Vec<String> = {
247            let mut v: Vec<String> = tokenizer
248                .get_added_tokens_decoder()
249                .values()
250                .filter(|t| t.special)
251                .map(|t| t.content.clone())
252                .collect();
253            v.sort();
254            v
255        };
256        assert_eq!(
257            specials,
258            vec!["<unk>".to_string(), "<|special_kept|>".to_string()],
259            "<|special_kept|> promoted; <|special_dropped|> stayed non-special"
260        );
261
262        // Decode round-trip assertion (the layer parser tests cannot
263        // reach): the registry mutation actually changes downstream
264        // decode behavior in the expected direction.
265        let enc_kept = tokenizer.encode("<|special_kept|>", false).unwrap();
266        let decoded_strip = tokenizer.decode(enc_kept.get_ids(), true).unwrap();
267        assert!(
268            !decoded_strip.contains("<|special_kept|>"),
269            "promoted special:true token must be stripped under skip_special_tokens=true; got {decoded_strip:?}"
270        );
271
272        let enc_drop = tokenizer.encode("<|special_dropped|>", false).unwrap();
273        let decoded_keep = tokenizer.decode(enc_drop.get_ids(), true).unwrap();
274        assert!(
275            decoded_keep.contains("<|special_dropped|>"),
276            "non-promoted special:false token must survive skip_special_tokens=true; got {decoded_keep:?}"
277        );
278    }
279}
dynamo_tokenizers/hf.rs

dynamo_tokenizers/
hf.rs
