1use std::{collections::HashSet, path::Path, sync::OnceLock};
5
6use tokenizers::tokenizer::{AddedToken, PostProcessor as _, Tokenizer as HfTokenizer};
7
8use super::{
9 Encoding, Error, Result, TokenIdType, TokenizerOptions,
10 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
11};
12
13pub struct HuggingFaceTokenizer {
14 tokenizer: HfTokenizer,
15 segment_tokenizers: OnceLock<(HfTokenizer, HfTokenizer)>,
16 byte_fallback_ids: HashSet<u32>,
17 special_ids: HashSet<u32>,
18 options: TokenizerOptions,
22}
23
24impl HuggingFaceTokenizer {
25 pub fn from_file(model_name: &str) -> Result<Self> {
30 let mut tokenizer = HfTokenizer::from_file(model_name)
31 .map_err(|err| Error::msg(format!("Error loading tokenizer: {}", err)))?;
32
33 if let Some(parent) = Path::new(model_name).parent() {
34 merge_special_tokens_from_config(&mut tokenizer, parent);
35 }
36
37 Ok(Self::from_tokenizer(tokenizer))
38 }
39
40 pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self {
41 let has_byte_fallback = tokenizer.get_decoder().is_some_and(|decoder| {
45 fn contains(value: &serde_json::Value) -> bool {
46 value["type"] == "ByteFallback"
47 || value["decoders"]
48 .as_array()
49 .is_some_and(|items| items.iter().any(contains))
50 }
51 serde_json::to_value(decoder).is_ok_and(|value| contains(&value))
52 });
53 let byte_fallback_ids = if has_byte_fallback {
54 tokenizer
55 .get_vocab(true)
56 .into_iter()
57 .filter_map(|(token, id)| {
58 (token.len() == 6
59 && token.starts_with("<0x")
60 && token.ends_with('>')
61 && u8::from_str_radix(&token[3..5], 16).is_ok())
62 .then_some(id)
63 })
64 .collect()
65 } else {
66 HashSet::new()
67 };
68 let special_ids = tokenizer
69 .get_added_tokens_decoder()
70 .into_iter()
71 .filter_map(|(id, token)| token.special.then_some(id))
72 .collect();
73 HuggingFaceTokenizer {
74 tokenizer,
75 segment_tokenizers: OnceLock::new(),
76 byte_fallback_ids,
77 special_ids,
78 options: TokenizerOptions::default(),
79 }
80 }
81
82 pub fn from_tokenizer_with_model_dir(tokenizer: HfTokenizer, model_dir: &Path) -> Self {
85 let mut tokenizer = tokenizer;
86 merge_special_tokens_from_config(&mut tokenizer, model_dir);
87 Self::from_tokenizer(tokenizer)
88 }
89}
90
91pub fn merge_special_tokens_from_config(tokenizer: &mut HfTokenizer, model_dir: &Path) {
100 let cfg_path = model_dir.join("tokenizer_config.json");
101 let Ok(raw) = std::fs::read_to_string(&cfg_path) else {
102 return;
103 };
104 let cfg: serde_json::Value = match serde_json::from_str(&raw) {
105 Ok(v) => v,
106 Err(e) => {
107 tracing::debug!(
108 target: "tokenizer",
109 path = %cfg_path.display(),
110 error = %e,
111 "tokenizer_config.json parse failed; skipping special-token merge"
112 );
113 return;
114 }
115 };
116 let Some(decoder) = cfg.get("added_tokens_decoder").and_then(|v| v.as_object()) else {
117 return;
118 };
119
120 let mut to_add: Vec<AddedToken> = Vec::new();
121 for (_id, spec) in decoder {
122 let obj = match spec.as_object() {
123 Some(o) => o,
124 None => continue,
125 };
126 if obj.get("special").and_then(|v| v.as_bool()) != Some(true) {
129 continue;
130 }
131 let Some(content) = obj.get("content").and_then(|v| v.as_str()) else {
132 continue;
133 };
134 if content.is_empty() {
135 continue;
136 }
137 let single_word = obj
138 .get("single_word")
139 .and_then(|v| v.as_bool())
140 .unwrap_or(false);
141 let lstrip = obj.get("lstrip").and_then(|v| v.as_bool()).unwrap_or(false);
142 let rstrip = obj.get("rstrip").and_then(|v| v.as_bool()).unwrap_or(false);
143 let normalized = obj
144 .get("normalized")
145 .and_then(|v| v.as_bool())
146 .unwrap_or(false);
147 let token = AddedToken::from(content.to_string(), true)
148 .single_word(single_word)
149 .lstrip(lstrip)
150 .rstrip(rstrip)
151 .normalized(normalized);
152 to_add.push(token);
153 }
154
155 if to_add.is_empty() {
156 return;
157 }
158 let added = match tokenizer.add_special_tokens(to_add.iter().cloned()) {
161 Ok(added) => added,
162 Err(e) => {
163 tracing::warn!(
164 target: "tokenizer",
165 path = %cfg_path.display(),
166 error = %e,
167 "special-token merge from tokenizer_config.json failed; skipping"
168 );
169 return;
170 }
171 };
172 if added > 0 {
173 let promoted: Vec<&str> = to_add.iter().map(|t| t.content.as_str()).collect();
179 tracing::warn!(
180 target: "tokenizer",
181 path = %cfg_path.display(),
182 added,
183 candidates = to_add.len(),
184 promoted = ?promoted,
185 "merged additional special tokens from tokenizer_config.json"
186 );
187 }
188}
189
190impl Encoder for HuggingFaceTokenizer {
191 fn encode(&self, input: &str) -> Result<Encoding> {
192 let encoding = self
194 .tokenizer
195 .encode(input, self.options.add_special_tokens)
196 .map_err(|err| Error::msg(format!("Error tokenizing input: {err}")))?;
197
198 Ok(Encoding::Hf(Box::new(encoding)))
199 }
200
201 fn encode_segments(&self, segments: &[super::EncodeSegment<'_>]) -> Result<Encoding> {
202 let (trusted, ordinary) = self.segment_tokenizers.get_or_init(|| {
203 let mut trusted = self.tokenizer.clone();
204 trusted.with_padding(None);
207 trusted
208 .with_truncation(None)
209 .expect("removing truncation cannot fail");
210 trusted.with_post_processor(None::<tokenizers::processors::PostProcessorWrapper>);
211 let mut ordinary = trusted.clone();
212 ordinary.set_encode_special_tokens(true);
216 (trusted, ordinary)
217 });
218 let encodings = segments
219 .iter()
220 .map(|segment| {
221 let tokenizer = if segment.allow_special {
222 trusted
223 } else {
224 ordinary
225 };
226 tokenizer
227 .encode(segment.text, false)
228 .map_err(|error| Error::msg(format!("Error tokenizing segment: {error}")))
229 })
230 .collect::<Result<Vec<_>>>()?;
231 let merged = tokenizers::Encoding::merge(encodings, true);
232 let encoding = self
233 .tokenizer
234 .post_process(merged, None, self.options.add_special_tokens)
235 .map_err(|error| Error::msg(format!("Error post-processing segments: {error}")))?;
236 Ok(Encoding::Hf(Box::new(encoding)))
237 }
238
239 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
240 let hf_encodings = self
241 .tokenizer
242 .encode_batch(inputs.to_vec(), self.options.add_special_tokens)
243 .map_err(|err| Error::msg(format!("Error batch tokenizing input: {err}")))?;
244
245 let encodings = hf_encodings
246 .into_iter()
247 .map(|enc| Encoding::Hf(Box::new(enc)))
248 .collect();
249
250 Ok(encodings)
251 }
252}
253
254impl Decoder for HuggingFaceTokenizer {
255 fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
256 token_ids
257 .iter()
258 .rev()
259 .find(|id| !skip_special_tokens || !self.special_ids.contains(id))
260 .is_some_and(|id| self.byte_fallback_ids.contains(id))
261 }
262
263 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
264 let text = self
266 .tokenizer
267 .decode(token_ids, skip_special_tokens)
268 .map_err(|err| Error::msg(format!("Error de-tokenizing input: {err}")))?;
269
270 Ok(text.into())
271 }
272}
273
274impl Tokenizer for HuggingFaceTokenizer {
275 fn validate_prefix_cache(&self) -> Result<()> {
276 if self.options.add_special_tokens {
277 return Err(Error::msg(
278 "HuggingFace tokenizers configured with add_special_tokens=true must remain uncached",
279 ));
280 }
281 Ok(())
282 }
283
284 fn with_options(mut self, options: TokenizerOptions) -> Self {
287 self.options = options;
288 self
289 }
290
291 fn vocab_size(&self) -> Option<usize> {
292 Some(self.tokenizer.get_vocab_size(true))
293 }
294
295 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
296 Ok(self.tokenizer.token_to_id(token))
297 }
298
299 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
300 let mut ids: Vec<TokenIdType> = self
301 .tokenizer
302 .get_added_tokens_decoder()
303 .into_iter()
304 .filter_map(|(id, token)| token.special.then_some(id))
305 .collect();
306 ids.sort_unstable();
307 Ok(ids)
308 }
309
310 fn num_special_tokens_added(&self) -> Result<usize> {
311 Ok(self
312 .tokenizer
313 .get_post_processor()
314 .map_or(0, |processor| processor.added_tokens(false)))
315 }
316}
317
318impl From<HfTokenizer> for HuggingFaceTokenizer {
319 fn from(tokenizer: HfTokenizer) -> Self {
320 Self::from_tokenizer(tokenizer)
321 }
322}
323
324#[cfg(test)]
325mod byte_fallback_stream_tests {
326 use super::*;
327
328 fn tokenizer() -> super::super::Tokenizer {
329 let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
330 "version": "1.0", "truncation": null, "padding": null,
331 "added_tokens": [{"id": 6, "content": "<eos>", "special": true,
332 "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}],
333 "normalizer": null, "pre_tokenizer": null, "post_processor": null,
334 "decoder": {"type": "Sequence", "decoders": [
335 {"type": "ByteFallback"}, {"type": "Fuse"}]},
336 "model": {"type": "BPE", "vocab": {
337 "<0x61>": 0, "<0xF5>": 1, "<0xC3>": 2, "<0xA9>": 3,
338 " hello": 4, "!": 5, "<eos>": 6}, "merges": [], "byte_fallback": true}
339 }))
340 .unwrap();
341 std::sync::Arc::new(HuggingFaceTokenizer::from_tokenizer(hf)).into()
342 }
343
344 #[test]
345 fn byte_runs_match_full_decode() {
346 let tokenizer = tokenizer();
347 for (ids, skip) in [
351 (vec![0, 1, 4], false),
352 (vec![2, 3, 1, 4], false),
353 (vec![2, 3, 5], false),
354 (vec![0], false),
355 (vec![2, 3], false),
356 (vec![2], false),
357 (vec![0, 1], false),
358 (vec![0, 6, 1, 4], false),
359 (vec![0, 6, 1, 4], true),
360 ] {
361 let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
362 let mut stream = tokenizer.decode_stream(&[], skip);
363 let mut actual = String::new();
364 for id in &ids {
365 actual.push_str(&stream.step(*id).unwrap().unwrap_or_default());
366 }
367 actual.push_str(&stream.finish().unwrap().unwrap_or_default());
368 assert_eq!(actual, expected, "ids={ids:?}, skip={skip}");
369 assert_eq!(stream.finish().unwrap(), None);
370 }
371 }
372
373 #[test]
374 fn only_byte_suffix_is_delayed_and_prompt_is_not_emitted() {
375 let tokenizer = tokenizer();
376 let mut stream = tokenizer.decode_stream(&[4], true);
377 assert_eq!(stream.step(5).unwrap().as_deref(), Some("!"));
378 assert_eq!(stream.step(0).unwrap(), None);
379 assert_eq!(stream.step(1).unwrap(), None);
380 assert_eq!(stream.step(4).unwrap().as_deref(), Some("�� hello"));
381 assert_eq!(stream.finish().unwrap(), None);
382 }
383}
384
385#[cfg(test)]
386mod tests {
387 use super::*;
396 use std::fs;
397 use tempfile::TempDir;
398
399 #[test]
400 fn merge_gate_round_trips_through_decode() {
401 const TOKENIZER_JSON: &str = r#"{
407 "version": "1.0",
408 "truncation": null,
409 "padding": null,
410 "added_tokens": [
411 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
412 ],
413 "normalizer": null,
414 "pre_tokenizer": null,
415 "post_processor": null,
416 "decoder": null,
417 "model": {
418 "type": "WordLevel",
419 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<|special_kept|>": 3, "<|special_dropped|>": 4},
420 "unk_token": "<unk>"
421 }
422 }"#;
423
424 const TOKENIZER_CONFIG_JSON: &str = r#"{
433 "added_tokens_decoder": {
434 "3": {"content": "<|special_kept|>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
435 "4": {"content": "<|special_dropped|>", "special": false, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
436 }
437 }"#;
438
439 let dir = TempDir::new().unwrap();
440 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
441 fs::write(
442 dir.path().join("tokenizer_config.json"),
443 TOKENIZER_CONFIG_JSON,
444 )
445 .unwrap();
446
447 let mut tokenizer = HfTokenizer::from_file(dir.path().join("tokenizer.json")).unwrap();
448 merge_special_tokens_from_config(&mut tokenizer, dir.path());
449
450 let specials: Vec<String> = {
452 let mut v: Vec<String> = tokenizer
453 .get_added_tokens_decoder()
454 .values()
455 .filter(|t| t.special)
456 .map(|t| t.content.clone())
457 .collect();
458 v.sort();
459 v
460 };
461 assert_eq!(
462 specials,
463 vec!["<unk>".to_string(), "<|special_kept|>".to_string()],
464 "<|special_kept|> promoted; <|special_dropped|> stayed non-special"
465 );
466
467 let enc_kept = tokenizer.encode("<|special_kept|>", false).unwrap();
471 let decoded_strip = tokenizer.decode(enc_kept.get_ids(), true).unwrap();
472 assert!(
473 !decoded_strip.contains("<|special_kept|>"),
474 "promoted special:true token must be stripped under skip_special_tokens=true; got {decoded_strip:?}"
475 );
476
477 let enc_drop = tokenizer.encode("<|special_dropped|>", false).unwrap();
478 let decoded_keep = tokenizer.decode(enc_drop.get_ids(), true).unwrap();
479 assert!(
480 decoded_keep.contains("<|special_dropped|>"),
481 "non-promoted special:false token must survive skip_special_tokens=true; got {decoded_keep:?}"
482 );
483 }
484
485 #[test]
486 fn add_special_tokens_flag_controls_encode() {
487 const TOKENIZER_JSON: &str = r#"{
492 "version": "1.0",
493 "truncation": null,
494 "padding": null,
495 "added_tokens": [
496 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
497 {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
498 ],
499 "normalizer": null,
500 "pre_tokenizer": null,
501 "post_processor": {
502 "type": "TemplateProcessing",
503 "single": [
504 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
505 {"Sequence": {"id": "A", "type_id": 0}}
506 ],
507 "pair": [
508 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
509 {"Sequence": {"id": "A", "type_id": 0}},
510 {"Sequence": {"id": "B", "type_id": 0}}
511 ],
512 "special_tokens": {
513 "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
514 }
515 },
516 "decoder": null,
517 "model": {
518 "type": "WordLevel",
519 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
520 "unk_token": "<unk>"
521 }
522 }"#;
523
524 let dir = TempDir::new().unwrap();
525 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
526 let path = dir.path().join("tokenizer.json");
527 let path = path.to_str().unwrap();
528
529 let ids = |enc: &Encoding| match enc {
530 Encoding::Hf(e) => e.get_ids().to_vec(),
531 _ => panic!("expected Hf encoding"),
532 };
533
534 let plain = HuggingFaceTokenizer::from_file(path).unwrap();
536 assert_eq!(ids(&plain.encode("hello").unwrap()), vec![1]);
537
538 let with_bos =
539 HuggingFaceTokenizer::from_file(path)
540 .unwrap()
541 .with_options(TokenizerOptions {
542 add_special_tokens: true,
543 });
544 assert_eq!(ids(&with_bos.encode("hello").unwrap()), vec![3, 1]);
545 let batch = with_bos.encode_batch(&["hello", "world"]).unwrap();
546 assert_eq!(ids(&batch[0]), vec![3, 1]);
547 assert_eq!(ids(&batch[1]), vec![3, 2]);
548
549 use crate::Tokenizer as TokenizerWrapper;
553 let wrapper_plain = TokenizerWrapper::from_file(path).unwrap();
554 assert_eq!(ids(&wrapper_plain.encode("hello").unwrap()), vec![1]);
555
556 let wrapper_bos = TokenizerWrapper::from_file_with_options(
557 path,
558 TokenizerOptions {
559 add_special_tokens: true,
560 },
561 )
562 .unwrap();
563 assert_eq!(ids(&wrapper_bos.encode("hello").unwrap()), vec![3, 1]);
564 }
565
566 #[test]
567 fn vocab_introspection_accessors() {
568 const TOKENIZER_JSON: &str = r#"{
569 "version": "1.0",
570 "truncation": null,
571 "padding": null,
572 "added_tokens": [
573 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
574 ],
575 "normalizer": null,
576 "pre_tokenizer": null,
577 "post_processor": null,
578 "decoder": null,
579 "model": {
580 "type": "WordLevel",
581 "vocab": {"<unk>": 0, "hello": 1, "world": 2},
582 "unk_token": "<unk>"
583 }
584 }"#;
585
586 let dir = TempDir::new().unwrap();
587 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
588 let path = dir.path().join("tokenizer.json");
589
590 let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
591 assert_eq!(tokenizer.vocab_size(), Some(3));
592 assert_eq!(tokenizer.token_to_id("hello").unwrap(), Some(1));
593 assert_eq!(tokenizer.special_token_ids().unwrap(), vec![0]);
594 assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 0);
595 }
596
597 #[test]
598 fn num_special_tokens_added_reflects_post_processor_additions() {
599 const TOKENIZER_JSON: &str = r#"{
600 "version": "1.0",
601 "truncation": null,
602 "padding": null,
603 "added_tokens": [
604 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
605 {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
606 ],
607 "normalizer": null,
608 "pre_tokenizer": null,
609 "post_processor": {
610 "type": "TemplateProcessing",
611 "single": [
612 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
613 {"Sequence": {"id": "A", "type_id": 0}}
614 ],
615 "pair": [
616 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
617 {"Sequence": {"id": "A", "type_id": 0}},
618 {"Sequence": {"id": "B", "type_id": 0}}
619 ],
620 "special_tokens": {
621 "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
622 }
623 },
624 "decoder": null,
625 "model": {
626 "type": "WordLevel",
627 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
628 "unk_token": "<unk>"
629 }
630 }"#;
631
632 let dir = TempDir::new().unwrap();
633 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
634 let path = dir.path().join("tokenizer.json");
635
636 let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
637 assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 1);
638 }
639}
640
641#[cfg(test)]
642mod segmented_tests {
643 use super::*;
644 use crate::EncodeSegment;
645
646 #[test]
647 fn segmented_hf_preserves_literal_controls_and_adds_bos_once() {
648 let mut vocab = serde_json::json!({"<unk>":0,"<|call|>":1,"<bos>":2});
649 for ch in "<|cal>".chars() {
650 let id = vocab.as_object().unwrap().len();
651 vocab[ch.to_string()] = serde_json::json!(id);
652 }
653 let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
654 "version":"1.0", "truncation":null, "padding":null,
655 "normalizer":null, "pre_tokenizer":null, "decoder":{"type":"Fuse"},
656 "added_tokens":[{"id":1,"content":"<|call|>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false},
657 {"id":2,"content":"<bos>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
658 "post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<bos>","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],
659 "pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"<bos>":{"id":"<bos>","ids":[2],"tokens":["<bos>"]}}},
660 "model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
661 })).unwrap();
662 for add_special_tokens in [false, true] {
663 let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf.clone())
664 .with_options(TokenizerOptions { add_special_tokens });
665 let encoding = tokenizer
666 .encode_segments(&[
667 EncodeSegment::control("<|call|>"),
668 EncodeSegment::ordinary("<|call|>"),
669 ])
670 .unwrap();
671 let ids = encoding.token_ids();
672 assert_eq!(ids.iter().filter(|id| **id == 1).count(), 1);
673 assert_eq!(
674 ids.iter().filter(|id| **id == 2).count(),
675 usize::from(add_special_tokens)
676 );
677 let decoded: String = tokenizer.decode(ids, true).unwrap().into();
678 assert_eq!(decoded, "<|call|>");
679 assert_eq!(
680 tokenizer
681 .encode("<|call|>")
682 .unwrap()
683 .token_ids()
684 .iter()
685 .filter(|id| **id == 1)
686 .count(),
687 1
688 );
689 }
690 }
691}