Skip to main content

dynamo_tokenizers/
hf.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{collections::HashSet, path::Path, sync::OnceLock};
5
6use tokenizers::tokenizer::{AddedToken, PostProcessor as _, Tokenizer as HfTokenizer};
7
8use super::{
9    Encoding, Error, Result, TokenIdType, TokenizerOptions,
10    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
11};
12
13pub struct HuggingFaceTokenizer {
14    tokenizer: HfTokenizer,
15    segment_tokenizers: OnceLock<(HfTokenizer, HfTokenizer)>,
16    byte_fallback_ids: HashSet<u32>,
17    special_ids: HashSet<u32>,
18    /// Construction-time options; see [`TokenizerOptions`]. Defaults preserve
19    /// the historical behavior (e.g. `add_special_tokens: false`). Set via
20    /// [`Tokenizer::with_options`].
21    options: TokenizerOptions,
22}
23
24impl HuggingFaceTokenizer {
25    /// Load from `tokenizer.json`, merging in special tokens declared only in
26    /// a sibling `tokenizer_config.json`'s `added_tokens_decoder`. Without
27    /// this, some releases (Qwen2-VL-2B's `<|image_pad|>`) BPE-shatter and
28    /// silently break MM-aware routing. The merge is idempotent.
29    pub fn from_file(model_name: &str) -> Result<Self> {
30        let mut tokenizer = HfTokenizer::from_file(model_name)
31            .map_err(|err| Error::msg(format!("Error loading tokenizer: {}", err)))?;
32
33        if let Some(parent) = Path::new(model_name).parent() {
34            merge_special_tokens_from_config(&mut tokenizer, parent);
35        }
36
37        Ok(Self::from_tokenizer(tokenizer))
38    }
39
40    pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self {
41        // ByteFallback decodes a contiguous run atomically: one invalid byte
42        // replaces the entire run, including previously valid ASCII/UTF-8.
43        // Discover the at-most-256 byte IDs once, not on the request hot path.
44        let has_byte_fallback = tokenizer.get_decoder().is_some_and(|decoder| {
45            fn contains(value: &serde_json::Value) -> bool {
46                value["type"] == "ByteFallback"
47                    || value["decoders"]
48                        .as_array()
49                        .is_some_and(|items| items.iter().any(contains))
50            }
51            serde_json::to_value(decoder).is_ok_and(|value| contains(&value))
52        });
53        let byte_fallback_ids = if has_byte_fallback {
54            tokenizer
55                .get_vocab(true)
56                .into_iter()
57                .filter_map(|(token, id)| {
58                    (token.len() == 6
59                        && token.starts_with("<0x")
60                        && token.ends_with('>')
61                        && u8::from_str_radix(&token[3..5], 16).is_ok())
62                    .then_some(id)
63                })
64                .collect()
65        } else {
66            HashSet::new()
67        };
68        let special_ids = tokenizer
69            .get_added_tokens_decoder()
70            .into_iter()
71            .filter_map(|(id, token)| token.special.then_some(id))
72            .collect();
73        HuggingFaceTokenizer {
74            tokenizer,
75            segment_tokenizers: OnceLock::new(),
76            byte_fallback_ids,
77            special_ids,
78            options: TokenizerOptions::default(),
79        }
80    }
81
82    /// Wrap an already-loaded `HfTokenizer` and merge in the sibling
83    /// `tokenizer_config.json` special tokens; see [`Self::from_file`].
84    pub fn from_tokenizer_with_model_dir(tokenizer: HfTokenizer, model_dir: &Path) -> Self {
85        let mut tokenizer = tokenizer;
86        merge_special_tokens_from_config(&mut tokenizer, model_dir);
87        Self::from_tokenizer(tokenizer)
88    }
89}
90
91/// Promote `tokenizer_config.json`'s `special: true` `added_tokens_decoder`
92/// entries onto `tokenizer`. Missing-file / parse errors are swallowed since
93/// the file is optional. See [`HuggingFaceTokenizer::from_file`].
94///
95/// `pub` so downstream crates (e.g. `dynamo-llm`'s model_card) can apply
96/// the same promotion before extracting the special-token boundary list
97/// for the L1 prefix cache — otherwise the cache and the wrapped
98/// tokenizer would disagree on which strings are atomic specials.
99pub fn merge_special_tokens_from_config(tokenizer: &mut HfTokenizer, model_dir: &Path) {
100    let cfg_path = model_dir.join("tokenizer_config.json");
101    let Ok(raw) = std::fs::read_to_string(&cfg_path) else {
102        return;
103    };
104    let cfg: serde_json::Value = match serde_json::from_str(&raw) {
105        Ok(v) => v,
106        Err(e) => {
107            tracing::debug!(
108                target: "tokenizer",
109                path = %cfg_path.display(),
110                error = %e,
111                "tokenizer_config.json parse failed; skipping special-token merge"
112            );
113            return;
114        }
115    };
116    let Some(decoder) = cfg.get("added_tokens_decoder").and_then(|v| v.as_object()) else {
117        return;
118    };
119
120    let mut to_add: Vec<AddedToken> = Vec::new();
121    for (_id, spec) in decoder {
122        let obj = match spec.as_object() {
123            Some(o) => o,
124            None => continue,
125        };
126        // The id is informational — `add_special_tokens` reuses the existing
127        // vocab id via `Model::token_to_id` on the content string.
128        if obj.get("special").and_then(|v| v.as_bool()) != Some(true) {
129            continue;
130        }
131        let Some(content) = obj.get("content").and_then(|v| v.as_str()) else {
132            continue;
133        };
134        if content.is_empty() {
135            continue;
136        }
137        let single_word = obj
138            .get("single_word")
139            .and_then(|v| v.as_bool())
140            .unwrap_or(false);
141        let lstrip = obj.get("lstrip").and_then(|v| v.as_bool()).unwrap_or(false);
142        let rstrip = obj.get("rstrip").and_then(|v| v.as_bool()).unwrap_or(false);
143        let normalized = obj
144            .get("normalized")
145            .and_then(|v| v.as_bool())
146            .unwrap_or(false);
147        let token = AddedToken::from(content.to_string(), true)
148            .single_word(single_word)
149            .lstrip(lstrip)
150            .rstrip(rstrip)
151            .normalized(normalized);
152        to_add.push(token);
153    }
154
155    if to_add.is_empty() {
156        return;
157    }
158    // Dedups against existing added-tokens, so this is a no-op when
159    // tokenizer.json already had them. Return value = net-new count.
160    let added = match tokenizer.add_special_tokens(to_add.iter().cloned()) {
161        Ok(added) => added,
162        Err(e) => {
163            tracing::warn!(
164                target: "tokenizer",
165                path = %cfg_path.display(),
166                error = %e,
167                "special-token merge from tokenizer_config.json failed; skipping"
168            );
169            return;
170        }
171    };
172    if added > 0 {
173        // Warn (not debug) when the merge actually promotes anything —
174        // intentionally loud so accidental promotion of a previously-
175        // non-special token shows up immediately in worker logs. Lists
176        // the literal token strings so debugging doesn't require a
177        // second pass through `added_tokens_decoder`.
178        let promoted: Vec<&str> = to_add.iter().map(|t| t.content.as_str()).collect();
179        tracing::warn!(
180            target: "tokenizer",
181            path = %cfg_path.display(),
182            added,
183            candidates = to_add.len(),
184            promoted = ?promoted,
185            "merged additional special tokens from tokenizer_config.json"
186        );
187    }
188}
189
190impl Encoder for HuggingFaceTokenizer {
191    fn encode(&self, input: &str) -> Result<Encoding> {
192        // This self.tokenizer is the library
193        let encoding = self
194            .tokenizer
195            .encode(input, self.options.add_special_tokens)
196            .map_err(|err| Error::msg(format!("Error tokenizing input: {err}")))?;
197
198        Ok(Encoding::Hf(Box::new(encoding)))
199    }
200
201    fn encode_segments(&self, segments: &[super::EncodeSegment<'_>]) -> Result<Encoding> {
202        let (trusted, ordinary) = self.segment_tokenizers.get_or_init(|| {
203            let mut trusted = self.tokenizer.clone();
204            // Apply padding, truncation and post-processing once to the joined
205            // sequence, rather than independently at every trust boundary.
206            trusted.with_padding(None);
207            trusted
208                .with_truncation(None)
209                .expect("removing truncation cannot fail");
210            trusted.with_post_processor(None::<tokenizers::processors::PostProcessorWrapper>);
211            let mut ordinary = trusted.clone();
212            // HF calls splitting special-token spellings into ordinary model
213            // tokens `encode_special_tokens`. Immutable clones keep concurrent
214            // trusted encodes independent of the vocabulary's splitting mode.
215            ordinary.set_encode_special_tokens(true);
216            (trusted, ordinary)
217        });
218        let encodings = segments
219            .iter()
220            .map(|segment| {
221                let tokenizer = if segment.allow_special {
222                    trusted
223                } else {
224                    ordinary
225                };
226                tokenizer
227                    .encode(segment.text, false)
228                    .map_err(|error| Error::msg(format!("Error tokenizing segment: {error}")))
229            })
230            .collect::<Result<Vec<_>>>()?;
231        let merged = tokenizers::Encoding::merge(encodings, true);
232        let encoding = self
233            .tokenizer
234            .post_process(merged, None, self.options.add_special_tokens)
235            .map_err(|error| Error::msg(format!("Error post-processing segments: {error}")))?;
236        Ok(Encoding::Hf(Box::new(encoding)))
237    }
238
239    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
240        let hf_encodings = self
241            .tokenizer
242            .encode_batch(inputs.to_vec(), self.options.add_special_tokens)
243            .map_err(|err| Error::msg(format!("Error batch tokenizing input: {err}")))?;
244
245        let encodings = hf_encodings
246            .into_iter()
247            .map(|enc| Encoding::Hf(Box::new(enc)))
248            .collect();
249
250        Ok(encodings)
251    }
252}
253
254impl Decoder for HuggingFaceTokenizer {
255    fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
256        token_ids
257            .iter()
258            .rev()
259            .find(|id| !skip_special_tokens || !self.special_ids.contains(id))
260            .is_some_and(|id| self.byte_fallback_ids.contains(id))
261    }
262
263    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
264        // This calls into the library
265        let text = self
266            .tokenizer
267            .decode(token_ids, skip_special_tokens)
268            .map_err(|err| Error::msg(format!("Error de-tokenizing input: {err}")))?;
269
270        Ok(text.into())
271    }
272}
273
274impl Tokenizer for HuggingFaceTokenizer {
275    fn validate_prefix_cache(&self) -> Result<()> {
276        if self.options.add_special_tokens {
277            return Err(Error::msg(
278                "HuggingFace tokenizers configured with add_special_tokens=true must remain uncached",
279            ));
280        }
281        // The cache joins separately encoded pieces, so per-call truncation or padding
282        // would apply to each piece instead of the whole input.
283        if self.tokenizer.get_truncation().is_some() || self.tokenizer.get_padding().is_some() {
284            return Err(Error::msg(
285                "HuggingFace tokenizers with truncation or padding must remain uncached",
286            ));
287        }
288        Ok(())
289    }
290
291    /// HF honors [`TokenizerOptions::add_special_tokens`] (BOS/EOS per the
292    /// tokenizer's post-processor template).
293    fn with_options(mut self, options: TokenizerOptions) -> Self {
294        self.options = options;
295        self
296    }
297
298    fn vocab_size(&self) -> Option<usize> {
299        Some(self.tokenizer.get_vocab_size(true))
300    }
301
302    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
303        Ok(self.tokenizer.token_to_id(token))
304    }
305
306    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
307        let mut ids: Vec<TokenIdType> = self
308            .tokenizer
309            .get_added_tokens_decoder()
310            .into_iter()
311            .filter_map(|(id, token)| token.special.then_some(id))
312            .collect();
313        ids.sort_unstable();
314        Ok(ids)
315    }
316
317    fn num_special_tokens_added(&self) -> Result<usize> {
318        Ok(self
319            .tokenizer
320            .get_post_processor()
321            .map_or(0, |processor| processor.added_tokens(false)))
322    }
323}
324
325impl From<HfTokenizer> for HuggingFaceTokenizer {
326    fn from(tokenizer: HfTokenizer) -> Self {
327        Self::from_tokenizer(tokenizer)
328    }
329}
330
331#[cfg(test)]
332mod byte_fallback_stream_tests {
333    use super::*;
334
335    fn tokenizer() -> super::super::Tokenizer {
336        let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
337            "version": "1.0", "truncation": null, "padding": null,
338            "added_tokens": [{"id": 6, "content": "<eos>", "special": true,
339                "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}],
340            "normalizer": null, "pre_tokenizer": null, "post_processor": null,
341            "decoder": {"type": "Sequence", "decoders": [
342                {"type": "ByteFallback"}, {"type": "Fuse"}]},
343            "model": {"type": "BPE", "vocab": {
344                "<0x61>": 0, "<0xF5>": 1, "<0xC3>": 2, "<0xA9>": 3,
345                " hello": 4, "!": 5, "<eos>": 6}, "merges": [], "byte_fallback": true}
346        }))
347        .unwrap();
348        std::sync::Arc::new(HuggingFaceTokenizer::from_tokenizer(hf)).into()
349    }
350
351    #[test]
352    fn byte_runs_match_full_decode() {
353        let tokenizer = tokenizer();
354        // Invalid tails rewrite the whole byte run, even previously valid ASCII
355        // or a complete multibyte character. Valid, incomplete and literal
356        // replacement output must all survive a terminal flush unchanged.
357        for (ids, skip) in [
358            (vec![0, 1, 4], false),
359            (vec![2, 3, 1, 4], false),
360            (vec![2, 3, 5], false),
361            (vec![0], false),
362            (vec![2, 3], false),
363            (vec![2], false),
364            (vec![0, 1], false),
365            (vec![0, 6, 1, 4], false),
366            (vec![0, 6, 1, 4], true),
367        ] {
368            let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
369            let mut stream = tokenizer.decode_stream(&[], skip);
370            let mut actual = String::new();
371            for id in &ids {
372                actual.push_str(&stream.step(*id).unwrap().unwrap_or_default());
373            }
374            actual.push_str(&stream.finish().unwrap().unwrap_or_default());
375            assert_eq!(actual, expected, "ids={ids:?}, skip={skip}");
376            assert_eq!(stream.finish().unwrap(), None);
377        }
378    }
379
380    #[test]
381    fn only_byte_suffix_is_delayed_and_prompt_is_not_emitted() {
382        let tokenizer = tokenizer();
383        let mut stream = tokenizer.decode_stream(&[4], true);
384        assert_eq!(stream.step(5).unwrap().as_deref(), Some("!"));
385        assert_eq!(stream.step(0).unwrap(), None);
386        assert_eq!(stream.step(1).unwrap(), None);
387        assert_eq!(stream.step(4).unwrap().as_deref(), Some("�� hello"));
388        assert_eq!(stream.finish().unwrap(), None);
389    }
390}
391
392#[cfg(test)]
393mod tests {
394    //! The existing tool-calling / reasoning parser tests inject already-
395    //! decoded text and never run `tokenizer.decode`, so they cannot
396    //! catch the class of bug where a non-special marker gets accidentally
397    //! promoted to "special" and silently disappears under
398    //! `skip_special_tokens=True`. One unit test pins both halves of the
399    //! gate (promote special:true / skip special:false) end-to-end through
400    //! the actual encode -> decode round trip — the layer above which the
401    //! parser tests cannot reach.
402    use super::*;
403    use std::fs;
404    use tempfile::TempDir;
405
406    #[test]
407    fn merge_gate_round_trips_through_decode() {
408        // Minimal WordLevel `tokenizer.json`. `<|special_kept|>` and
409        // `<|special_dropped|>` are deliberately NOT pre-declared as
410        // special in `added_tokens` here — that's exactly the shape
411        // `merge_special_tokens_from_config` is supposed to fix from
412        // tokenizer_config.json.
413        const TOKENIZER_JSON: &str = r#"{
414            "version": "1.0",
415            "truncation": null,
416            "padding": null,
417            "added_tokens": [
418                {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
419            ],
420            "normalizer": null,
421            "pre_tokenizer": null,
422            "post_processor": null,
423            "decoder": null,
424            "model": {
425                "type": "WordLevel",
426                "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<|special_kept|>": 3, "<|special_dropped|>": 4},
427                "unk_token": "<unk>"
428            }
429        }"#;
430
431        // `<|special_kept|>` is `special: true` → must be promoted, and
432        // therefore stripped under skip_special_tokens=true.
433        // `<|special_dropped|>` is `special: false` → must be skipped,
434        // and therefore survive skip_special_tokens=true. The latter is
435        // the Ryan/Keiven concern: tool-call / reasoning markers are
436        // universally declared `special: false` precisely so the parser
437        // still sees them; a regression here would silently break
438        // parsing without any of the parser unit tests turning red.
439        const TOKENIZER_CONFIG_JSON: &str = r#"{
440            "added_tokens_decoder": {
441                "3": {"content": "<|special_kept|>",    "special": true,  "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
442                "4": {"content": "<|special_dropped|>", "special": false, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
443            }
444        }"#;
445
446        let dir = TempDir::new().unwrap();
447        fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
448        fs::write(
449            dir.path().join("tokenizer_config.json"),
450            TOKENIZER_CONFIG_JSON,
451        )
452        .unwrap();
453
454        let mut tokenizer = HfTokenizer::from_file(dir.path().join("tokenizer.json")).unwrap();
455        merge_special_tokens_from_config(&mut tokenizer, dir.path());
456
457        // Registry assertion: only the special:true entry was promoted.
458        let specials: Vec<String> = {
459            let mut v: Vec<String> = tokenizer
460                .get_added_tokens_decoder()
461                .values()
462                .filter(|t| t.special)
463                .map(|t| t.content.clone())
464                .collect();
465            v.sort();
466            v
467        };
468        assert_eq!(
469            specials,
470            vec!["<unk>".to_string(), "<|special_kept|>".to_string()],
471            "<|special_kept|> promoted; <|special_dropped|> stayed non-special"
472        );
473
474        // Decode round-trip assertion (the layer parser tests cannot
475        // reach): the registry mutation actually changes downstream
476        // decode behavior in the expected direction.
477        let enc_kept = tokenizer.encode("<|special_kept|>", false).unwrap();
478        let decoded_strip = tokenizer.decode(enc_kept.get_ids(), true).unwrap();
479        assert!(
480            !decoded_strip.contains("<|special_kept|>"),
481            "promoted special:true token must be stripped under skip_special_tokens=true; got {decoded_strip:?}"
482        );
483
484        let enc_drop = tokenizer.encode("<|special_dropped|>", false).unwrap();
485        let decoded_keep = tokenizer.decode(enc_drop.get_ids(), true).unwrap();
486        assert!(
487            decoded_keep.contains("<|special_dropped|>"),
488            "non-promoted special:false token must survive skip_special_tokens=true; got {decoded_keep:?}"
489        );
490    }
491
492    #[test]
493    fn add_special_tokens_flag_controls_encode() {
494        // WordLevel tokenizer with a TemplateProcessing post-processor that
495        // prepends <bos>. The post-processor only fires when encode is
496        // called with add_special_tokens=true, which is exactly the knob
497        // `TokenizerOptions`/`with_options` exposes.
498        const TOKENIZER_JSON: &str = r#"{
499            "version": "1.0",
500            "truncation": null,
501            "padding": null,
502            "added_tokens": [
503                {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
504                {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
505            ],
506            "normalizer": null,
507            "pre_tokenizer": null,
508            "post_processor": {
509                "type": "TemplateProcessing",
510                "single": [
511                    {"SpecialToken": {"id": "<bos>", "type_id": 0}},
512                    {"Sequence": {"id": "A", "type_id": 0}}
513                ],
514                "pair": [
515                    {"SpecialToken": {"id": "<bos>", "type_id": 0}},
516                    {"Sequence": {"id": "A", "type_id": 0}},
517                    {"Sequence": {"id": "B", "type_id": 0}}
518                ],
519                "special_tokens": {
520                    "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
521                }
522            },
523            "decoder": null,
524            "model": {
525                "type": "WordLevel",
526                "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
527                "unk_token": "<unk>"
528            }
529        }"#;
530
531        let dir = TempDir::new().unwrap();
532        fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
533        let path = dir.path().join("tokenizer.json");
534        let path = path.to_str().unwrap();
535
536        let ids = |enc: &Encoding| match enc {
537            Encoding::Hf(e) => e.get_ids().to_vec(),
538            _ => panic!("expected Hf encoding"),
539        };
540
541        // Default: unchanged historical behavior — no BOS.
542        let plain = HuggingFaceTokenizer::from_file(path).unwrap();
543        assert_eq!(ids(&plain.encode("hello").unwrap()), vec![1]);
544
545        let with_bos =
546            HuggingFaceTokenizer::from_file(path)
547                .unwrap()
548                .with_options(TokenizerOptions {
549                    add_special_tokens: true,
550                });
551        assert_eq!(ids(&with_bos.encode("hello").unwrap()), vec![3, 1]);
552        let batch = with_bos.encode_batch(&["hello", "world"]).unwrap();
553        assert_eq!(ids(&batch[0]), vec![3, 1]);
554        assert_eq!(ids(&batch[1]), vec![3, 2]);
555
556        // Same knob through the top-level wrapper: from_file keeps the
557        // historical default, from_file_with_options threads the flag to
558        // construction before the tokenizer goes behind the shared Arc.
559        use crate::Tokenizer as TokenizerWrapper;
560        let wrapper_plain = TokenizerWrapper::from_file(path).unwrap();
561        assert_eq!(ids(&wrapper_plain.encode("hello").unwrap()), vec![1]);
562
563        let wrapper_bos = TokenizerWrapper::from_file_with_options(
564            path,
565            TokenizerOptions {
566                add_special_tokens: true,
567            },
568        )
569        .unwrap();
570        assert_eq!(ids(&wrapper_bos.encode("hello").unwrap()), vec![3, 1]);
571    }
572
573    #[test]
574    fn truncation_or_padding_blocks_prefix_cache() {
575        let path = concat!(
576            env!("CARGO_MANIFEST_DIR"),
577            "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
578        );
579        let mut truncated = HfTokenizer::from_file(path).unwrap();
580        truncated
581            .with_truncation(Some(tokenizers::TruncationParams::default()))
582            .unwrap();
583        let mut padded = HfTokenizer::from_file(path).unwrap();
584        padded.with_padding(Some(tokenizers::PaddingParams::default()));
585        for hf in [truncated, padded] {
586            assert!(
587                HuggingFaceTokenizer::from_tokenizer(hf)
588                    .validate_prefix_cache()
589                    .is_err()
590            );
591        }
592    }
593
594    #[test]
595    fn vocab_introspection_accessors() {
596        const TOKENIZER_JSON: &str = r#"{
597            "version": "1.0",
598            "truncation": null,
599            "padding": null,
600            "added_tokens": [
601                {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
602            ],
603            "normalizer": null,
604            "pre_tokenizer": null,
605            "post_processor": null,
606            "decoder": null,
607            "model": {
608                "type": "WordLevel",
609                "vocab": {"<unk>": 0, "hello": 1, "world": 2},
610                "unk_token": "<unk>"
611            }
612        }"#;
613
614        let dir = TempDir::new().unwrap();
615        fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
616        let path = dir.path().join("tokenizer.json");
617
618        let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
619        assert_eq!(tokenizer.vocab_size(), Some(3));
620        assert_eq!(tokenizer.token_to_id("hello").unwrap(), Some(1));
621        assert_eq!(tokenizer.special_token_ids().unwrap(), vec![0]);
622        assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 0);
623    }
624
625    #[test]
626    fn num_special_tokens_added_reflects_post_processor_additions() {
627        const TOKENIZER_JSON: &str = r#"{
628            "version": "1.0",
629            "truncation": null,
630            "padding": null,
631            "added_tokens": [
632                {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
633                {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
634            ],
635            "normalizer": null,
636            "pre_tokenizer": null,
637            "post_processor": {
638                "type": "TemplateProcessing",
639                "single": [
640                    {"SpecialToken": {"id": "<bos>", "type_id": 0}},
641                    {"Sequence": {"id": "A", "type_id": 0}}
642                ],
643                "pair": [
644                    {"SpecialToken": {"id": "<bos>", "type_id": 0}},
645                    {"Sequence": {"id": "A", "type_id": 0}},
646                    {"Sequence": {"id": "B", "type_id": 0}}
647                ],
648                "special_tokens": {
649                    "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
650                }
651            },
652            "decoder": null,
653            "model": {
654                "type": "WordLevel",
655                "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
656                "unk_token": "<unk>"
657            }
658        }"#;
659
660        let dir = TempDir::new().unwrap();
661        fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
662        let path = dir.path().join("tokenizer.json");
663
664        let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
665        assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 1);
666    }
667}
668
669#[cfg(test)]
670mod segmented_tests {
671    use super::*;
672    use crate::EncodeSegment;
673
674    #[test]
675    fn segmented_hf_preserves_literal_controls_and_adds_bos_once() {
676        let mut vocab = serde_json::json!({"<unk>":0,"<|call|>":1,"<bos>":2});
677        for ch in "<|cal>".chars() {
678            let id = vocab.as_object().unwrap().len();
679            vocab[ch.to_string()] = serde_json::json!(id);
680        }
681        let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
682            "version":"1.0", "truncation":null, "padding":null,
683            "normalizer":null, "pre_tokenizer":null, "decoder":{"type":"Fuse"},
684            "added_tokens":[{"id":1,"content":"<|call|>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false},
685                {"id":2,"content":"<bos>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
686            "post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<bos>","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],
687                "pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"<bos>":{"id":"<bos>","ids":[2],"tokens":["<bos>"]}}},
688            "model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
689        })).unwrap();
690        for add_special_tokens in [false, true] {
691            let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf.clone())
692                .with_options(TokenizerOptions { add_special_tokens });
693            let encoding = tokenizer
694                .encode_segments(&[
695                    EncodeSegment::control("<|call|>"),
696                    EncodeSegment::ordinary("<|call|>"),
697                ])
698                .unwrap();
699            let ids = encoding.token_ids();
700            assert_eq!(ids.iter().filter(|id| **id == 1).count(), 1);
701            assert_eq!(
702                ids.iter().filter(|id| **id == 2).count(),
703                usize::from(add_special_tokens)
704            );
705            let decoded: String = tokenizer.decode(ids, true).unwrap().into();
706            assert_eq!(decoded, "<|call|>");
707            assert_eq!(
708                tokenizer
709                    .encode("<|call|>")
710                    .unwrap()
711                    .token_ids()
712                    .iter()
713                    .filter(|id| **id == 1)
714                    .count(),
715                1
716            );
717        }
718    }
719}