1use std::{collections::HashSet, path::Path, sync::OnceLock};
5
6use tokenizers::tokenizer::{AddedToken, PostProcessor as _, Tokenizer as HfTokenizer};
7
8use super::{
9 Encoding, Error, Result, TokenIdType, TokenizerOptions,
10 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
11};
12
13pub struct HuggingFaceTokenizer {
14 tokenizer: HfTokenizer,
15 segment_tokenizers: OnceLock<(HfTokenizer, HfTokenizer)>,
16 byte_fallback_ids: HashSet<u32>,
17 special_ids: HashSet<u32>,
18 options: TokenizerOptions,
22}
23
24impl HuggingFaceTokenizer {
25 pub fn from_file(model_name: &str) -> Result<Self> {
30 let mut tokenizer = HfTokenizer::from_file(model_name)
31 .map_err(|err| Error::msg(format!("Error loading tokenizer: {}", err)))?;
32
33 if let Some(parent) = Path::new(model_name).parent() {
34 merge_special_tokens_from_config(&mut tokenizer, parent);
35 }
36
37 Ok(Self::from_tokenizer(tokenizer))
38 }
39
40 pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self {
41 let has_byte_fallback = tokenizer.get_decoder().is_some_and(|decoder| {
45 fn contains(value: &serde_json::Value) -> bool {
46 value["type"] == "ByteFallback"
47 || value["decoders"]
48 .as_array()
49 .is_some_and(|items| items.iter().any(contains))
50 }
51 serde_json::to_value(decoder).is_ok_and(|value| contains(&value))
52 });
53 let byte_fallback_ids = if has_byte_fallback {
54 tokenizer
55 .get_vocab(true)
56 .into_iter()
57 .filter_map(|(token, id)| {
58 (token.len() == 6
59 && token.starts_with("<0x")
60 && token.ends_with('>')
61 && u8::from_str_radix(&token[3..5], 16).is_ok())
62 .then_some(id)
63 })
64 .collect()
65 } else {
66 HashSet::new()
67 };
68 let special_ids = tokenizer
69 .get_added_tokens_decoder()
70 .into_iter()
71 .filter_map(|(id, token)| token.special.then_some(id))
72 .collect();
73 HuggingFaceTokenizer {
74 tokenizer,
75 segment_tokenizers: OnceLock::new(),
76 byte_fallback_ids,
77 special_ids,
78 options: TokenizerOptions::default(),
79 }
80 }
81
82 pub fn from_tokenizer_with_model_dir(tokenizer: HfTokenizer, model_dir: &Path) -> Self {
85 let mut tokenizer = tokenizer;
86 merge_special_tokens_from_config(&mut tokenizer, model_dir);
87 Self::from_tokenizer(tokenizer)
88 }
89}
90
91pub fn merge_special_tokens_from_config(tokenizer: &mut HfTokenizer, model_dir: &Path) {
100 let cfg_path = model_dir.join("tokenizer_config.json");
101 let Ok(raw) = std::fs::read_to_string(&cfg_path) else {
102 return;
103 };
104 let cfg: serde_json::Value = match serde_json::from_str(&raw) {
105 Ok(v) => v,
106 Err(e) => {
107 tracing::debug!(
108 target: "tokenizer",
109 path = %cfg_path.display(),
110 error = %e,
111 "tokenizer_config.json parse failed; skipping special-token merge"
112 );
113 return;
114 }
115 };
116 let Some(decoder) = cfg.get("added_tokens_decoder").and_then(|v| v.as_object()) else {
117 return;
118 };
119
120 let mut to_add: Vec<AddedToken> = Vec::new();
121 for (_id, spec) in decoder {
122 let obj = match spec.as_object() {
123 Some(o) => o,
124 None => continue,
125 };
126 if obj.get("special").and_then(|v| v.as_bool()) != Some(true) {
129 continue;
130 }
131 let Some(content) = obj.get("content").and_then(|v| v.as_str()) else {
132 continue;
133 };
134 if content.is_empty() {
135 continue;
136 }
137 let single_word = obj
138 .get("single_word")
139 .and_then(|v| v.as_bool())
140 .unwrap_or(false);
141 let lstrip = obj.get("lstrip").and_then(|v| v.as_bool()).unwrap_or(false);
142 let rstrip = obj.get("rstrip").and_then(|v| v.as_bool()).unwrap_or(false);
143 let normalized = obj
144 .get("normalized")
145 .and_then(|v| v.as_bool())
146 .unwrap_or(false);
147 let token = AddedToken::from(content.to_string(), true)
148 .single_word(single_word)
149 .lstrip(lstrip)
150 .rstrip(rstrip)
151 .normalized(normalized);
152 to_add.push(token);
153 }
154
155 if to_add.is_empty() {
156 return;
157 }
158 let added = tokenizer.add_special_tokens(&to_add);
161 if added > 0 {
162 let promoted: Vec<&str> = to_add.iter().map(|t| t.content.as_str()).collect();
168 tracing::warn!(
169 target: "tokenizer",
170 path = %cfg_path.display(),
171 added,
172 candidates = to_add.len(),
173 promoted = ?promoted,
174 "merged additional special tokens from tokenizer_config.json"
175 );
176 }
177}
178
179impl Encoder for HuggingFaceTokenizer {
180 fn encode(&self, input: &str) -> Result<Encoding> {
181 let encoding = self
183 .tokenizer
184 .encode(input, self.options.add_special_tokens)
185 .map_err(|err| Error::msg(format!("Error tokenizing input: {err}")))?;
186
187 Ok(Encoding::Hf(Box::new(encoding)))
188 }
189
190 fn encode_segments(&self, segments: &[super::EncodeSegment<'_>]) -> Result<Encoding> {
191 let (trusted, ordinary) = self.segment_tokenizers.get_or_init(|| {
192 let mut trusted = self.tokenizer.clone();
193 trusted.with_padding(None);
196 trusted
197 .with_truncation(None)
198 .expect("removing truncation cannot fail");
199 trusted.with_post_processor(None::<tokenizers::processors::PostProcessorWrapper>);
200 let mut ordinary = trusted.clone();
201 ordinary.set_encode_special_tokens(true);
205 (trusted, ordinary)
206 });
207 let encodings = segments
208 .iter()
209 .map(|segment| {
210 let tokenizer = if segment.allow_special {
211 trusted
212 } else {
213 ordinary
214 };
215 tokenizer
216 .encode(segment.text, false)
217 .map_err(|error| Error::msg(format!("Error tokenizing segment: {error}")))
218 })
219 .collect::<Result<Vec<_>>>()?;
220 let merged = tokenizers::Encoding::merge(encodings, true);
221 let encoding = self
222 .tokenizer
223 .post_process(merged, None, self.options.add_special_tokens)
224 .map_err(|error| Error::msg(format!("Error post-processing segments: {error}")))?;
225 Ok(Encoding::Hf(Box::new(encoding)))
226 }
227
228 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
229 let hf_encodings = self
230 .tokenizer
231 .encode_batch(inputs.to_vec(), self.options.add_special_tokens)
232 .map_err(|err| Error::msg(format!("Error batch tokenizing input: {err}")))?;
233
234 let encodings = hf_encodings
235 .into_iter()
236 .map(|enc| Encoding::Hf(Box::new(enc)))
237 .collect();
238
239 Ok(encodings)
240 }
241}
242
243impl Decoder for HuggingFaceTokenizer {
244 fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
245 token_ids
246 .iter()
247 .rev()
248 .find(|id| !skip_special_tokens || !self.special_ids.contains(id))
249 .is_some_and(|id| self.byte_fallback_ids.contains(id))
250 }
251
252 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
253 let text = self
255 .tokenizer
256 .decode(token_ids, skip_special_tokens)
257 .map_err(|err| Error::msg(format!("Error de-tokenizing input: {err}")))?;
258
259 Ok(text.into())
260 }
261}
262
263impl Tokenizer for HuggingFaceTokenizer {
264 fn validate_prefix_cache(&self) -> Result<()> {
265 if self.options.add_special_tokens {
266 return Err(Error::msg(
267 "HuggingFace tokenizers configured with add_special_tokens=true must remain uncached",
268 ));
269 }
270 Ok(())
271 }
272
273 fn with_options(mut self, options: TokenizerOptions) -> Self {
276 self.options = options;
277 self
278 }
279
280 fn vocab_size(&self) -> Option<usize> {
281 Some(self.tokenizer.get_vocab_size(true))
282 }
283
284 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
285 Ok(self.tokenizer.token_to_id(token))
286 }
287
288 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
289 let mut ids: Vec<TokenIdType> = self
290 .tokenizer
291 .get_added_tokens_decoder()
292 .into_iter()
293 .filter_map(|(id, token)| token.special.then_some(id))
294 .collect();
295 ids.sort_unstable();
296 Ok(ids)
297 }
298
299 fn num_special_tokens_added(&self) -> Result<usize> {
300 Ok(self
301 .tokenizer
302 .get_post_processor()
303 .map_or(0, |processor| processor.added_tokens(false)))
304 }
305}
306
307impl From<HfTokenizer> for HuggingFaceTokenizer {
308 fn from(tokenizer: HfTokenizer) -> Self {
309 Self::from_tokenizer(tokenizer)
310 }
311}
312
313#[cfg(test)]
314mod byte_fallback_stream_tests {
315 use super::*;
316
317 fn tokenizer() -> super::super::Tokenizer {
318 let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
319 "version": "1.0", "truncation": null, "padding": null,
320 "added_tokens": [{"id": 6, "content": "<eos>", "special": true,
321 "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}],
322 "normalizer": null, "pre_tokenizer": null, "post_processor": null,
323 "decoder": {"type": "Sequence", "decoders": [
324 {"type": "ByteFallback"}, {"type": "Fuse"}]},
325 "model": {"type": "BPE", "vocab": {
326 "<0x61>": 0, "<0xF5>": 1, "<0xC3>": 2, "<0xA9>": 3,
327 " hello": 4, "!": 5, "<eos>": 6}, "merges": [], "byte_fallback": true}
328 }))
329 .unwrap();
330 std::sync::Arc::new(HuggingFaceTokenizer::from_tokenizer(hf)).into()
331 }
332
333 #[test]
334 fn byte_runs_match_full_decode() {
335 let tokenizer = tokenizer();
336 for (ids, skip) in [
340 (vec![0, 1, 4], false),
341 (vec![2, 3, 1, 4], false),
342 (vec![2, 3, 5], false),
343 (vec![0], false),
344 (vec![2, 3], false),
345 (vec![2], false),
346 (vec![0, 1], false),
347 (vec![0, 6, 1, 4], false),
348 (vec![0, 6, 1, 4], true),
349 ] {
350 let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
351 let mut stream = tokenizer.decode_stream(&[], skip);
352 let mut actual = String::new();
353 for id in &ids {
354 actual.push_str(&stream.step(*id).unwrap().unwrap_or_default());
355 }
356 actual.push_str(&stream.finish().unwrap().unwrap_or_default());
357 assert_eq!(actual, expected, "ids={ids:?}, skip={skip}");
358 assert_eq!(stream.finish().unwrap(), None);
359 }
360 }
361
362 #[test]
363 fn only_byte_suffix_is_delayed_and_prompt_is_not_emitted() {
364 let tokenizer = tokenizer();
365 let mut stream = tokenizer.decode_stream(&[4], true);
366 assert_eq!(stream.step(5).unwrap().as_deref(), Some("!"));
367 assert_eq!(stream.step(0).unwrap(), None);
368 assert_eq!(stream.step(1).unwrap(), None);
369 assert_eq!(stream.step(4).unwrap().as_deref(), Some("�� hello"));
370 assert_eq!(stream.finish().unwrap(), None);
371 }
372}
373
374#[cfg(test)]
375mod tests {
376 use super::*;
385 use std::fs;
386 use tempfile::TempDir;
387
388 #[test]
389 fn merge_gate_round_trips_through_decode() {
390 const TOKENIZER_JSON: &str = r#"{
396 "version": "1.0",
397 "truncation": null,
398 "padding": null,
399 "added_tokens": [
400 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
401 ],
402 "normalizer": null,
403 "pre_tokenizer": null,
404 "post_processor": null,
405 "decoder": null,
406 "model": {
407 "type": "WordLevel",
408 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<|special_kept|>": 3, "<|special_dropped|>": 4},
409 "unk_token": "<unk>"
410 }
411 }"#;
412
413 const TOKENIZER_CONFIG_JSON: &str = r#"{
422 "added_tokens_decoder": {
423 "3": {"content": "<|special_kept|>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
424 "4": {"content": "<|special_dropped|>", "special": false, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
425 }
426 }"#;
427
428 let dir = TempDir::new().unwrap();
429 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
430 fs::write(
431 dir.path().join("tokenizer_config.json"),
432 TOKENIZER_CONFIG_JSON,
433 )
434 .unwrap();
435
436 let mut tokenizer = HfTokenizer::from_file(dir.path().join("tokenizer.json")).unwrap();
437 merge_special_tokens_from_config(&mut tokenizer, dir.path());
438
439 let specials: Vec<String> = {
441 let mut v: Vec<String> = tokenizer
442 .get_added_tokens_decoder()
443 .values()
444 .filter(|t| t.special)
445 .map(|t| t.content.clone())
446 .collect();
447 v.sort();
448 v
449 };
450 assert_eq!(
451 specials,
452 vec!["<unk>".to_string(), "<|special_kept|>".to_string()],
453 "<|special_kept|> promoted; <|special_dropped|> stayed non-special"
454 );
455
456 let enc_kept = tokenizer.encode("<|special_kept|>", false).unwrap();
460 let decoded_strip = tokenizer.decode(enc_kept.get_ids(), true).unwrap();
461 assert!(
462 !decoded_strip.contains("<|special_kept|>"),
463 "promoted special:true token must be stripped under skip_special_tokens=true; got {decoded_strip:?}"
464 );
465
466 let enc_drop = tokenizer.encode("<|special_dropped|>", false).unwrap();
467 let decoded_keep = tokenizer.decode(enc_drop.get_ids(), true).unwrap();
468 assert!(
469 decoded_keep.contains("<|special_dropped|>"),
470 "non-promoted special:false token must survive skip_special_tokens=true; got {decoded_keep:?}"
471 );
472 }
473
474 #[test]
475 fn add_special_tokens_flag_controls_encode() {
476 const TOKENIZER_JSON: &str = r#"{
481 "version": "1.0",
482 "truncation": null,
483 "padding": null,
484 "added_tokens": [
485 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
486 {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
487 ],
488 "normalizer": null,
489 "pre_tokenizer": null,
490 "post_processor": {
491 "type": "TemplateProcessing",
492 "single": [
493 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
494 {"Sequence": {"id": "A", "type_id": 0}}
495 ],
496 "pair": [
497 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
498 {"Sequence": {"id": "A", "type_id": 0}},
499 {"Sequence": {"id": "B", "type_id": 0}}
500 ],
501 "special_tokens": {
502 "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
503 }
504 },
505 "decoder": null,
506 "model": {
507 "type": "WordLevel",
508 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
509 "unk_token": "<unk>"
510 }
511 }"#;
512
513 let dir = TempDir::new().unwrap();
514 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
515 let path = dir.path().join("tokenizer.json");
516 let path = path.to_str().unwrap();
517
518 let ids = |enc: &Encoding| match enc {
519 Encoding::Hf(e) => e.get_ids().to_vec(),
520 _ => panic!("expected Hf encoding"),
521 };
522
523 let plain = HuggingFaceTokenizer::from_file(path).unwrap();
525 assert_eq!(ids(&plain.encode("hello").unwrap()), vec![1]);
526
527 let with_bos =
528 HuggingFaceTokenizer::from_file(path)
529 .unwrap()
530 .with_options(TokenizerOptions {
531 add_special_tokens: true,
532 });
533 assert_eq!(ids(&with_bos.encode("hello").unwrap()), vec![3, 1]);
534 let batch = with_bos.encode_batch(&["hello", "world"]).unwrap();
535 assert_eq!(ids(&batch[0]), vec![3, 1]);
536 assert_eq!(ids(&batch[1]), vec![3, 2]);
537
538 use crate::Tokenizer as TokenizerWrapper;
542 let wrapper_plain = TokenizerWrapper::from_file(path).unwrap();
543 assert_eq!(ids(&wrapper_plain.encode("hello").unwrap()), vec![1]);
544
545 let wrapper_bos = TokenizerWrapper::from_file_with_options(
546 path,
547 TokenizerOptions {
548 add_special_tokens: true,
549 },
550 )
551 .unwrap();
552 assert_eq!(ids(&wrapper_bos.encode("hello").unwrap()), vec![3, 1]);
553 }
554
555 #[test]
556 fn vocab_introspection_accessors() {
557 const TOKENIZER_JSON: &str = r#"{
558 "version": "1.0",
559 "truncation": null,
560 "padding": null,
561 "added_tokens": [
562 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
563 ],
564 "normalizer": null,
565 "pre_tokenizer": null,
566 "post_processor": null,
567 "decoder": null,
568 "model": {
569 "type": "WordLevel",
570 "vocab": {"<unk>": 0, "hello": 1, "world": 2},
571 "unk_token": "<unk>"
572 }
573 }"#;
574
575 let dir = TempDir::new().unwrap();
576 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
577 let path = dir.path().join("tokenizer.json");
578
579 let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
580 assert_eq!(tokenizer.vocab_size(), Some(3));
581 assert_eq!(tokenizer.token_to_id("hello").unwrap(), Some(1));
582 assert_eq!(tokenizer.special_token_ids().unwrap(), vec![0]);
583 assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 0);
584 }
585
586 #[test]
587 fn num_special_tokens_added_reflects_post_processor_additions() {
588 const TOKENIZER_JSON: &str = r#"{
589 "version": "1.0",
590 "truncation": null,
591 "padding": null,
592 "added_tokens": [
593 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
594 {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
595 ],
596 "normalizer": null,
597 "pre_tokenizer": null,
598 "post_processor": {
599 "type": "TemplateProcessing",
600 "single": [
601 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
602 {"Sequence": {"id": "A", "type_id": 0}}
603 ],
604 "pair": [
605 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
606 {"Sequence": {"id": "A", "type_id": 0}},
607 {"Sequence": {"id": "B", "type_id": 0}}
608 ],
609 "special_tokens": {
610 "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
611 }
612 },
613 "decoder": null,
614 "model": {
615 "type": "WordLevel",
616 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
617 "unk_token": "<unk>"
618 }
619 }"#;
620
621 let dir = TempDir::new().unwrap();
622 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
623 let path = dir.path().join("tokenizer.json");
624
625 let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
626 assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 1);
627 }
628}
629
630#[cfg(test)]
631mod segmented_tests {
632 use super::*;
633 use crate::EncodeSegment;
634
635 #[test]
636 fn segmented_hf_preserves_literal_controls_and_adds_bos_once() {
637 let mut vocab = serde_json::json!({"<unk>":0,"<|call|>":1,"<bos>":2});
638 for ch in "<|cal>".chars() {
639 let id = vocab.as_object().unwrap().len();
640 vocab[ch.to_string()] = serde_json::json!(id);
641 }
642 let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
643 "version":"1.0", "truncation":null, "padding":null,
644 "normalizer":null, "pre_tokenizer":null, "decoder":{"type":"Fuse"},
645 "added_tokens":[{"id":1,"content":"<|call|>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false},
646 {"id":2,"content":"<bos>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
647 "post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<bos>","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],
648 "pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"<bos>":{"id":"<bos>","ids":[2],"tokens":["<bos>"]}}},
649 "model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
650 })).unwrap();
651 for add_special_tokens in [false, true] {
652 let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf.clone())
653 .with_options(TokenizerOptions { add_special_tokens });
654 let encoding = tokenizer
655 .encode_segments(&[
656 EncodeSegment::control("<|call|>"),
657 EncodeSegment::ordinary("<|call|>"),
658 ])
659 .unwrap();
660 let ids = encoding.token_ids();
661 assert_eq!(ids.iter().filter(|id| **id == 1).count(), 1);
662 assert_eq!(
663 ids.iter().filter(|id| **id == 2).count(),
664 usize::from(add_special_tokens)
665 );
666 let decoded: String = tokenizer.decode(ids, true).unwrap().into();
667 assert_eq!(decoded, "<|call|>");
668 assert_eq!(
669 tokenizer
670 .encode("<|call|>")
671 .unwrap()
672 .token_ids()
673 .iter()
674 .filter(|id| **id == 1)
675 .count(),
676 1
677 );
678 }
679 }
680}