1use std::{collections::HashSet, path::Path, sync::OnceLock};
5
6use tokenizers::tokenizer::{
7 AddedToken, AddedVocabulary, PostProcessor as _, Tokenizer as HfTokenizer,
8};
9
10use super::{
11 Encoding, Error, Result, TokenIdType, TokenizerOptions,
12 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
13};
14
15pub struct HuggingFaceTokenizer {
16 tokenizer: HfTokenizer,
17 segment_tokenizers: OnceLock<(HfTokenizer, HfTokenizer)>,
18 byte_fallback_ids: HashSet<u32>,
19 special_ids: HashSet<u32>,
20 added_ids: HashSet<u32>,
24 options: TokenizerOptions,
28}
29
30impl HuggingFaceTokenizer {
31 pub(crate) fn added_ids(&self) -> &HashSet<u32> {
33 &self.added_ids
34 }
35
36 pub fn from_file(model_name: &str) -> Result<Self> {
41 let mut tokenizer = HfTokenizer::from_file(model_name)
42 .map_err(|err| Error::msg(format!("Error loading tokenizer: {}", err)))?;
43
44 if let Some(parent) = Path::new(model_name).parent() {
45 merge_special_tokens_from_config(&mut tokenizer, parent);
46 }
47
48 Ok(Self::from_tokenizer(tokenizer))
49 }
50
51 pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self {
52 let has_byte_fallback = tokenizer.get_decoder().is_some_and(|decoder| {
56 fn contains(value: &serde_json::Value) -> bool {
57 value["type"] == "ByteFallback"
58 || value["decoders"]
59 .as_array()
60 .is_some_and(|items| items.iter().any(contains))
61 }
62 serde_json::to_value(decoder).is_ok_and(|value| contains(&value))
63 });
64 let byte_fallback_ids = if has_byte_fallback {
65 tokenizer
66 .get_vocab(true)
67 .into_iter()
68 .filter_map(|(token, id)| {
69 (token.len() == 6
70 && token.starts_with("<0x")
71 && token.ends_with('>')
72 && u8::from_str_radix(&token[3..5], 16).is_ok())
73 .then_some(id)
74 })
75 .collect()
76 } else {
77 HashSet::new()
78 };
79 let added_tokens = tokenizer.get_added_tokens_decoder();
80 let special_ids = added_tokens
81 .iter()
82 .filter_map(|(id, token)| token.special.then_some(*id))
83 .collect();
84 let added_ids = added_tokens.into_keys().collect();
85 HuggingFaceTokenizer {
86 tokenizer,
87 segment_tokenizers: OnceLock::new(),
88 byte_fallback_ids,
89 special_ids,
90 added_ids,
91 options: TokenizerOptions::default(),
92 }
93 }
94
95 pub fn from_tokenizer_with_model_dir(tokenizer: HfTokenizer, model_dir: &Path) -> Self {
98 let mut tokenizer = tokenizer;
99 merge_special_tokens_from_config(&mut tokenizer, model_dir);
100 Self::from_tokenizer(tokenizer)
101 }
102}
103
104pub fn merge_special_tokens_from_config(tokenizer: &mut HfTokenizer, model_dir: &Path) {
113 let cfg_path = model_dir.join("tokenizer_config.json");
114 let Ok(raw) = std::fs::read_to_string(&cfg_path) else {
115 return;
116 };
117 let cfg: serde_json::Value = match serde_json::from_str(&raw) {
118 Ok(v) => v,
119 Err(e) => {
120 tracing::debug!(
121 target: "tokenizer",
122 path = %cfg_path.display(),
123 error = %e,
124 "tokenizer_config.json parse failed; skipping special-token merge"
125 );
126 return;
127 }
128 };
129 let Some(decoder) = cfg.get("added_tokens_decoder").and_then(|v| v.as_object()) else {
130 return;
131 };
132
133 let mut to_add: Vec<AddedToken> = Vec::new();
134 for (_id, spec) in decoder {
135 let obj = match spec.as_object() {
136 Some(o) => o,
137 None => continue,
138 };
139 if obj.get("special").and_then(|v| v.as_bool()) != Some(true) {
142 continue;
143 }
144 let Some(content) = obj.get("content").and_then(|v| v.as_str()) else {
145 continue;
146 };
147 if content.is_empty() {
148 continue;
149 }
150 let single_word = obj
151 .get("single_word")
152 .and_then(|v| v.as_bool())
153 .unwrap_or(false);
154 let lstrip = obj.get("lstrip").and_then(|v| v.as_bool()).unwrap_or(false);
155 let rstrip = obj.get("rstrip").and_then(|v| v.as_bool()).unwrap_or(false);
156 let normalized = obj
157 .get("normalized")
158 .and_then(|v| v.as_bool())
159 .unwrap_or(false);
160 let token = AddedToken::from(content.to_string(), true)
161 .single_word(single_word)
162 .lstrip(lstrip)
163 .rstrip(rstrip)
164 .normalized(normalized);
165 to_add.push(token);
166 }
167
168 if to_add.is_empty() {
169 return;
170 }
171 let added = match tokenizer.add_special_tokens(to_add.iter().cloned()) {
174 Ok(added) => added,
175 Err(e) => {
176 tracing::warn!(
177 target: "tokenizer",
178 path = %cfg_path.display(),
179 error = %e,
180 "special-token merge from tokenizer_config.json failed; skipping"
181 );
182 return;
183 }
184 };
185 if added > 0 {
186 let promoted: Vec<&str> = to_add.iter().map(|t| t.content.as_str()).collect();
192 tracing::warn!(
193 target: "tokenizer",
194 path = %cfg_path.display(),
195 added,
196 candidates = to_add.len(),
197 promoted = ?promoted,
198 "merged additional special tokens from tokenizer_config.json"
199 );
200 }
201}
202
203impl Encoder for HuggingFaceTokenizer {
204 fn encode(&self, input: &str) -> Result<Encoding> {
205 let encoding = self
207 .tokenizer
208 .encode(input, self.options.add_special_tokens)
209 .map_err(|err| Error::msg(format!("Error tokenizing input: {err}")))?;
210
211 Ok(Encoding::Hf(Box::new(encoding)))
212 }
213
214 fn encode_segments(&self, segments: &[super::EncodeSegment<'_>]) -> Result<Encoding> {
215 let (trusted, ordinary) = self.segment_tokenizers.get_or_init(|| {
216 let mut trusted = self.tokenizer.clone();
217 trusted.with_padding(None);
220 trusted
221 .with_truncation(None)
222 .expect("removing truncation cannot fail");
223 trusted.with_post_processor(None::<tokenizers::processors::PostProcessorWrapper>);
224 let mut ordinary = trusted.clone();
228 ordinary.with_added_vocabulary(AddedVocabulary::new());
229 (trusted, ordinary)
230 });
231 let encodings = segments
232 .iter()
233 .map(|segment| {
234 let tokenizer = if segment.allow_special {
235 trusted
236 } else {
237 ordinary
238 };
239 let encoding = tokenizer
240 .encode(segment.text, false)
241 .map_err(|error| Error::msg(format!("Error tokenizing segment: {error}")))?;
242 if !segment.allow_special {
243 crate::reject_control_ids_in_ordinary_segment(
244 encoding.get_ids(),
245 &self.added_ids,
246 )?;
247 }
248 Ok(encoding)
249 })
250 .collect::<Result<Vec<_>>>()?;
251 let merged = tokenizers::Encoding::merge(encodings, true);
252 let encoding = self
253 .tokenizer
254 .post_process(merged, None, self.options.add_special_tokens)
255 .map_err(|error| Error::msg(format!("Error post-processing segments: {error}")))?;
256 Ok(Encoding::Hf(Box::new(encoding)))
257 }
258
259 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
260 let hf_encodings = self
261 .tokenizer
262 .encode_batch(inputs.to_vec(), self.options.add_special_tokens)
263 .map_err(|err| Error::msg(format!("Error batch tokenizing input: {err}")))?;
264
265 let encodings = hf_encodings
266 .into_iter()
267 .map(|enc| Encoding::Hf(Box::new(enc)))
268 .collect();
269
270 Ok(encodings)
271 }
272}
273
274impl Decoder for HuggingFaceTokenizer {
275 fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
276 token_ids
277 .iter()
278 .rev()
279 .find(|id| !skip_special_tokens || !self.special_ids.contains(id))
280 .is_some_and(|id| self.byte_fallback_ids.contains(id))
281 }
282
283 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
284 let text = self
286 .tokenizer
287 .decode(token_ids, skip_special_tokens)
288 .map_err(|err| Error::msg(format!("Error de-tokenizing input: {err}")))?;
289
290 Ok(text.into())
291 }
292}
293
294impl Tokenizer for HuggingFaceTokenizer {
295 fn validate_prefix_cache(&self) -> Result<()> {
296 if self.options.add_special_tokens {
297 return Err(Error::msg(
298 "HuggingFace tokenizers configured with add_special_tokens=true must remain uncached",
299 ));
300 }
301 if self.tokenizer.get_truncation().is_some() || self.tokenizer.get_padding().is_some() {
304 return Err(Error::msg(
305 "HuggingFace tokenizers with truncation or padding must remain uncached",
306 ));
307 }
308 Ok(())
309 }
310
311 fn validate_segment_prefix_cache(&self) -> Result<()> {
314 self.validate_prefix_cache()
315 }
316
317 fn with_options(mut self, options: TokenizerOptions) -> Self {
320 self.options = options;
321 self
322 }
323
324 fn vocab_size(&self) -> Option<usize> {
325 Some(self.tokenizer.get_vocab_size(true))
326 }
327
328 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
329 Ok(self.tokenizer.token_to_id(token))
330 }
331
332 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
333 let mut ids: Vec<TokenIdType> = self
334 .tokenizer
335 .get_added_tokens_decoder()
336 .into_iter()
337 .filter_map(|(id, token)| token.special.then_some(id))
338 .collect();
339 ids.sort_unstable();
340 Ok(ids)
341 }
342
343 fn num_special_tokens_added(&self) -> Result<usize> {
344 Ok(self
345 .tokenizer
346 .get_post_processor()
347 .map_or(0, |processor| processor.added_tokens(false)))
348 }
349}
350
351impl From<HfTokenizer> for HuggingFaceTokenizer {
352 fn from(tokenizer: HfTokenizer) -> Self {
353 Self::from_tokenizer(tokenizer)
354 }
355}
356
357#[cfg(test)]
358mod byte_fallback_stream_tests {
359 use super::*;
360
361 fn tokenizer() -> super::super::Tokenizer {
362 let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
363 "version": "1.0", "truncation": null, "padding": null,
364 "added_tokens": [{"id": 6, "content": "<eos>", "special": true,
365 "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}],
366 "normalizer": null, "pre_tokenizer": null, "post_processor": null,
367 "decoder": {"type": "Sequence", "decoders": [
368 {"type": "ByteFallback"}, {"type": "Fuse"}]},
369 "model": {"type": "BPE", "vocab": {
370 "<0x61>": 0, "<0xF5>": 1, "<0xC3>": 2, "<0xA9>": 3,
371 " hello": 4, "!": 5, "<eos>": 6}, "merges": [], "byte_fallback": true}
372 }))
373 .unwrap();
374 std::sync::Arc::new(HuggingFaceTokenizer::from_tokenizer(hf)).into()
375 }
376
377 #[test]
378 fn byte_runs_match_full_decode() {
379 let tokenizer = tokenizer();
380 for (ids, skip) in [
384 (vec![0, 1, 4], false),
385 (vec![2, 3, 1, 4], false),
386 (vec![2, 3, 5], false),
387 (vec![0], false),
388 (vec![2, 3], false),
389 (vec![2], false),
390 (vec![0, 1], false),
391 (vec![0, 6, 1, 4], false),
392 (vec![0, 6, 1, 4], true),
393 ] {
394 let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
395 let mut stream = tokenizer.decode_stream(&[], skip);
396 let mut actual = String::new();
397 for id in &ids {
398 actual.push_str(&stream.step(*id).unwrap().unwrap_or_default());
399 }
400 actual.push_str(&stream.finish().unwrap().unwrap_or_default());
401 assert_eq!(actual, expected, "ids={ids:?}, skip={skip}");
402 assert_eq!(stream.finish().unwrap(), None);
403 }
404 }
405
406 #[test]
407 fn only_byte_suffix_is_delayed_and_prompt_is_not_emitted() {
408 let tokenizer = tokenizer();
409 let mut stream = tokenizer.decode_stream(&[4], true);
410 assert_eq!(stream.step(5).unwrap().as_deref(), Some("!"));
411 assert_eq!(stream.step(0).unwrap(), None);
412 assert_eq!(stream.step(1).unwrap(), None);
413 assert_eq!(stream.step(4).unwrap().as_deref(), Some("�� hello"));
414 assert_eq!(stream.finish().unwrap(), None);
415 }
416}
417
418#[cfg(test)]
419mod tests {
420 use super::*;
429 use std::fs;
430 use tempfile::TempDir;
431
432 #[test]
433 fn merge_gate_round_trips_through_decode() {
434 const TOKENIZER_JSON: &str = r#"{
440 "version": "1.0",
441 "truncation": null,
442 "padding": null,
443 "added_tokens": [
444 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
445 ],
446 "normalizer": null,
447 "pre_tokenizer": null,
448 "post_processor": null,
449 "decoder": null,
450 "model": {
451 "type": "WordLevel",
452 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<|special_kept|>": 3, "<|special_dropped|>": 4},
453 "unk_token": "<unk>"
454 }
455 }"#;
456
457 const TOKENIZER_CONFIG_JSON: &str = r#"{
466 "added_tokens_decoder": {
467 "3": {"content": "<|special_kept|>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
468 "4": {"content": "<|special_dropped|>", "special": false, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
469 }
470 }"#;
471
472 let dir = TempDir::new().unwrap();
473 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
474 fs::write(
475 dir.path().join("tokenizer_config.json"),
476 TOKENIZER_CONFIG_JSON,
477 )
478 .unwrap();
479
480 let mut tokenizer = HfTokenizer::from_file(dir.path().join("tokenizer.json")).unwrap();
481 merge_special_tokens_from_config(&mut tokenizer, dir.path());
482
483 let specials: Vec<String> = {
485 let mut v: Vec<String> = tokenizer
486 .get_added_tokens_decoder()
487 .values()
488 .filter(|t| t.special)
489 .map(|t| t.content.clone())
490 .collect();
491 v.sort();
492 v
493 };
494 assert_eq!(
495 specials,
496 vec!["<unk>".to_string(), "<|special_kept|>".to_string()],
497 "<|special_kept|> promoted; <|special_dropped|> stayed non-special"
498 );
499
500 let enc_kept = tokenizer.encode("<|special_kept|>", false).unwrap();
504 let decoded_strip = tokenizer.decode(enc_kept.get_ids(), true).unwrap();
505 assert!(
506 !decoded_strip.contains("<|special_kept|>"),
507 "promoted special:true token must be stripped under skip_special_tokens=true; got {decoded_strip:?}"
508 );
509
510 let enc_drop = tokenizer.encode("<|special_dropped|>", false).unwrap();
511 let decoded_keep = tokenizer.decode(enc_drop.get_ids(), true).unwrap();
512 assert!(
513 decoded_keep.contains("<|special_dropped|>"),
514 "non-promoted special:false token must survive skip_special_tokens=true; got {decoded_keep:?}"
515 );
516 }
517
518 #[test]
519 fn add_special_tokens_flag_controls_encode() {
520 const TOKENIZER_JSON: &str = r#"{
525 "version": "1.0",
526 "truncation": null,
527 "padding": null,
528 "added_tokens": [
529 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
530 {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
531 ],
532 "normalizer": null,
533 "pre_tokenizer": null,
534 "post_processor": {
535 "type": "TemplateProcessing",
536 "single": [
537 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
538 {"Sequence": {"id": "A", "type_id": 0}}
539 ],
540 "pair": [
541 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
542 {"Sequence": {"id": "A", "type_id": 0}},
543 {"Sequence": {"id": "B", "type_id": 0}}
544 ],
545 "special_tokens": {
546 "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
547 }
548 },
549 "decoder": null,
550 "model": {
551 "type": "WordLevel",
552 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
553 "unk_token": "<unk>"
554 }
555 }"#;
556
557 let dir = TempDir::new().unwrap();
558 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
559 let path = dir.path().join("tokenizer.json");
560 let path = path.to_str().unwrap();
561
562 let ids = |enc: &Encoding| match enc {
563 Encoding::Hf(e) => e.get_ids().to_vec(),
564 _ => panic!("expected Hf encoding"),
565 };
566
567 let plain = HuggingFaceTokenizer::from_file(path).unwrap();
569 assert_eq!(ids(&plain.encode("hello").unwrap()), vec![1]);
570
571 let with_bos =
572 HuggingFaceTokenizer::from_file(path)
573 .unwrap()
574 .with_options(TokenizerOptions {
575 add_special_tokens: true,
576 });
577 assert_eq!(ids(&with_bos.encode("hello").unwrap()), vec![3, 1]);
578 let batch = with_bos.encode_batch(&["hello", "world"]).unwrap();
579 assert_eq!(ids(&batch[0]), vec![3, 1]);
580 assert_eq!(ids(&batch[1]), vec![3, 2]);
581
582 use crate::Tokenizer as TokenizerWrapper;
586 let wrapper_plain = TokenizerWrapper::from_file(path).unwrap();
587 assert_eq!(ids(&wrapper_plain.encode("hello").unwrap()), vec![1]);
588
589 let wrapper_bos = TokenizerWrapper::from_file_with_options(
590 path,
591 TokenizerOptions {
592 add_special_tokens: true,
593 },
594 )
595 .unwrap();
596 assert_eq!(ids(&wrapper_bos.encode("hello").unwrap()), vec![3, 1]);
597 }
598
599 #[test]
600 fn truncation_or_padding_blocks_prefix_cache() {
601 let path = concat!(
602 env!("CARGO_MANIFEST_DIR"),
603 "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
604 );
605 let mut truncated = HfTokenizer::from_file(path).unwrap();
606 truncated
607 .with_truncation(Some(tokenizers::TruncationParams::default()))
608 .unwrap();
609 let mut padded = HfTokenizer::from_file(path).unwrap();
610 padded.with_padding(Some(tokenizers::PaddingParams::default()));
611 for hf in [truncated, padded] {
612 assert!(
613 HuggingFaceTokenizer::from_tokenizer(hf)
614 .validate_prefix_cache()
615 .is_err()
616 );
617 }
618 }
619
620 #[test]
621 fn vocab_introspection_accessors() {
622 const TOKENIZER_JSON: &str = r#"{
623 "version": "1.0",
624 "truncation": null,
625 "padding": null,
626 "added_tokens": [
627 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
628 ],
629 "normalizer": null,
630 "pre_tokenizer": null,
631 "post_processor": null,
632 "decoder": null,
633 "model": {
634 "type": "WordLevel",
635 "vocab": {"<unk>": 0, "hello": 1, "world": 2},
636 "unk_token": "<unk>"
637 }
638 }"#;
639
640 let dir = TempDir::new().unwrap();
641 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
642 let path = dir.path().join("tokenizer.json");
643
644 let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
645 assert_eq!(tokenizer.vocab_size(), Some(3));
646 assert_eq!(tokenizer.token_to_id("hello").unwrap(), Some(1));
647 assert_eq!(tokenizer.special_token_ids().unwrap(), vec![0]);
648 assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 0);
649 }
650
651 #[test]
652 fn num_special_tokens_added_reflects_post_processor_additions() {
653 const TOKENIZER_JSON: &str = r#"{
654 "version": "1.0",
655 "truncation": null,
656 "padding": null,
657 "added_tokens": [
658 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
659 {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
660 ],
661 "normalizer": null,
662 "pre_tokenizer": null,
663 "post_processor": {
664 "type": "TemplateProcessing",
665 "single": [
666 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
667 {"Sequence": {"id": "A", "type_id": 0}}
668 ],
669 "pair": [
670 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
671 {"Sequence": {"id": "A", "type_id": 0}},
672 {"Sequence": {"id": "B", "type_id": 0}}
673 ],
674 "special_tokens": {
675 "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
676 }
677 },
678 "decoder": null,
679 "model": {
680 "type": "WordLevel",
681 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
682 "unk_token": "<unk>"
683 }
684 }"#;
685
686 let dir = TempDir::new().unwrap();
687 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
688 let path = dir.path().join("tokenizer.json");
689
690 let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
691 assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 1);
692 }
693}
694
695#[cfg(test)]
696mod segmented_tests {
697 use super::*;
698 use crate::EncodeSegment;
699
700 #[test]
701 fn segmented_hf_preserves_literal_controls_and_adds_bos_once() {
702 let mut vocab = serde_json::json!({"<unk>":0,"<|call|>":1,"<bos>":2});
703 for ch in "<|cal>".chars() {
704 let id = vocab.as_object().unwrap().len();
705 vocab[ch.to_string()] = serde_json::json!(id);
706 }
707 let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
708 "version":"1.0", "truncation":null, "padding":null,
709 "normalizer":null, "pre_tokenizer":null, "decoder":{"type":"Fuse"},
710 "added_tokens":[{"id":1,"content":"<|call|>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false},
711 {"id":2,"content":"<bos>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
712 "post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<bos>","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],
713 "pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"<bos>":{"id":"<bos>","ids":[2],"tokens":["<bos>"]}}},
714 "model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
715 })).unwrap();
716 for add_special_tokens in [false, true] {
717 let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf.clone())
718 .with_options(TokenizerOptions { add_special_tokens });
719 let encoding = tokenizer
720 .encode_segments(&[
721 EncodeSegment::control("<|call|>"),
722 EncodeSegment::ordinary("<|call|>"),
723 ])
724 .unwrap();
725 let ids = encoding.token_ids();
726 assert_eq!(ids.iter().filter(|id| **id == 1).count(), 1);
727 assert_eq!(
728 ids.iter().filter(|id| **id == 2).count(),
729 usize::from(add_special_tokens)
730 );
731 let decoded: String = tokenizer.decode(ids, true).unwrap().into();
732 assert_eq!(decoded, "<|call|>");
733 assert_eq!(
734 tokenizer
735 .encode("<|call|>")
736 .unwrap()
737 .token_ids()
738 .iter()
739 .filter(|id| **id == 1)
740 .count(),
741 1
742 );
743 }
744 }
745 #[test]
746 fn segmented_hf_encodes_non_special_added_tokens_as_text() {
747 let mut vocab = serde_json::json!({"<unk>":0,"<think>":1});
750 for ch in "<think>".chars() {
751 let id = vocab.as_object().unwrap().len();
752 vocab[ch.to_string()] = serde_json::json!(id);
753 }
754 let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
755 "version":"1.0", "truncation":null, "padding":null,
756 "normalizer":null, "pre_tokenizer":null, "post_processor":null, "decoder":{"type":"Fuse"},
757 "added_tokens":[{"id":1,"content":"<think>","special":false,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
758 "model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
759 })).unwrap();
760 let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf);
761 let ids = |segment| {
762 tokenizer
763 .encode_segments(&[segment])
764 .unwrap()
765 .token_ids()
766 .to_vec()
767 };
768 assert_eq!(ids(EncodeSegment::control("<think>")), [1]);
769 let ordinary = ids(EncodeSegment::ordinary("<think>"));
770 assert_eq!(ordinary.len(), "<think>".len());
771 assert!(!ordinary.contains(&1));
772 }
773
774 #[test]
775 fn segmented_hf_rejects_ordinary_control_ids() {
776 let mut json: serde_json::Value = serde_json::from_str(
779 &std::fs::read_to_string(concat!(
780 env!("CARGO_MANIFEST_DIR"),
781 "/tests/data/minimal-bpe/tokenizer.json"
782 ))
783 .unwrap(),
784 )
785 .unwrap();
786 json["added_tokens"] = serde_json::json!([{
787 "id": 18,
788 "content": "He",
789 "single_word": false,
790 "lstrip": false,
791 "rstrip": false,
792 "normalized": false,
793 "special": true
794 }]);
795 let tokenizer = HuggingFaceTokenizer::from_tokenizer(serde_json::from_value(json).unwrap());
796 let control = tokenizer
797 .encode_segments(&[EncodeSegment::control("He")])
798 .unwrap();
799 assert_eq!(control.token_ids(), &[18]);
800 let error = tokenizer
801 .encode_segments(&[EncodeSegment::ordinary("He")])
802 .unwrap_err();
803 assert!(
804 error.to_string().contains("control-token id 18"),
805 "unexpected error: {error}"
806 );
807 }
808}