1use std::{collections::HashSet, path::Path, sync::OnceLock};
5
6use tokenizers::tokenizer::{AddedToken, PostProcessor as _, Tokenizer as HfTokenizer};
7
8use super::{
9 Encoding, Error, Result, TokenIdType, TokenizerOptions,
10 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
11};
12
13pub struct HuggingFaceTokenizer {
14 tokenizer: HfTokenizer,
15 segment_tokenizers: OnceLock<(HfTokenizer, HfTokenizer)>,
16 byte_fallback_ids: HashSet<u32>,
17 special_ids: HashSet<u32>,
18 options: TokenizerOptions,
22}
23
24impl HuggingFaceTokenizer {
25 pub fn from_file(model_name: &str) -> Result<Self> {
30 let mut tokenizer = HfTokenizer::from_file(model_name)
31 .map_err(|err| Error::msg(format!("Error loading tokenizer: {}", err)))?;
32
33 if let Some(parent) = Path::new(model_name).parent() {
34 merge_special_tokens_from_config(&mut tokenizer, parent);
35 }
36
37 Ok(Self::from_tokenizer(tokenizer))
38 }
39
40 pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self {
41 let has_byte_fallback = tokenizer.get_decoder().is_some_and(|decoder| {
45 fn contains(value: &serde_json::Value) -> bool {
46 value["type"] == "ByteFallback"
47 || value["decoders"]
48 .as_array()
49 .is_some_and(|items| items.iter().any(contains))
50 }
51 serde_json::to_value(decoder).is_ok_and(|value| contains(&value))
52 });
53 let byte_fallback_ids = if has_byte_fallback {
54 tokenizer
55 .get_vocab(true)
56 .into_iter()
57 .filter_map(|(token, id)| {
58 (token.len() == 6
59 && token.starts_with("<0x")
60 && token.ends_with('>')
61 && u8::from_str_radix(&token[3..5], 16).is_ok())
62 .then_some(id)
63 })
64 .collect()
65 } else {
66 HashSet::new()
67 };
68 let special_ids = tokenizer
69 .get_added_tokens_decoder()
70 .into_iter()
71 .filter_map(|(id, token)| token.special.then_some(id))
72 .collect();
73 HuggingFaceTokenizer {
74 tokenizer,
75 segment_tokenizers: OnceLock::new(),
76 byte_fallback_ids,
77 special_ids,
78 options: TokenizerOptions::default(),
79 }
80 }
81
82 pub fn from_tokenizer_with_model_dir(tokenizer: HfTokenizer, model_dir: &Path) -> Self {
85 let mut tokenizer = tokenizer;
86 merge_special_tokens_from_config(&mut tokenizer, model_dir);
87 Self::from_tokenizer(tokenizer)
88 }
89}
90
91pub fn merge_special_tokens_from_config(tokenizer: &mut HfTokenizer, model_dir: &Path) {
100 let cfg_path = model_dir.join("tokenizer_config.json");
101 let Ok(raw) = std::fs::read_to_string(&cfg_path) else {
102 return;
103 };
104 let cfg: serde_json::Value = match serde_json::from_str(&raw) {
105 Ok(v) => v,
106 Err(e) => {
107 tracing::debug!(
108 target: "tokenizer",
109 path = %cfg_path.display(),
110 error = %e,
111 "tokenizer_config.json parse failed; skipping special-token merge"
112 );
113 return;
114 }
115 };
116 let Some(decoder) = cfg.get("added_tokens_decoder").and_then(|v| v.as_object()) else {
117 return;
118 };
119
120 let mut to_add: Vec<AddedToken> = Vec::new();
121 for (_id, spec) in decoder {
122 let obj = match spec.as_object() {
123 Some(o) => o,
124 None => continue,
125 };
126 if obj.get("special").and_then(|v| v.as_bool()) != Some(true) {
129 continue;
130 }
131 let Some(content) = obj.get("content").and_then(|v| v.as_str()) else {
132 continue;
133 };
134 if content.is_empty() {
135 continue;
136 }
137 let single_word = obj
138 .get("single_word")
139 .and_then(|v| v.as_bool())
140 .unwrap_or(false);
141 let lstrip = obj.get("lstrip").and_then(|v| v.as_bool()).unwrap_or(false);
142 let rstrip = obj.get("rstrip").and_then(|v| v.as_bool()).unwrap_or(false);
143 let normalized = obj
144 .get("normalized")
145 .and_then(|v| v.as_bool())
146 .unwrap_or(false);
147 let token = AddedToken::from(content.to_string(), true)
148 .single_word(single_word)
149 .lstrip(lstrip)
150 .rstrip(rstrip)
151 .normalized(normalized);
152 to_add.push(token);
153 }
154
155 if to_add.is_empty() {
156 return;
157 }
158 let added = match tokenizer.add_special_tokens(to_add.iter().cloned()) {
161 Ok(added) => added,
162 Err(e) => {
163 tracing::warn!(
164 target: "tokenizer",
165 path = %cfg_path.display(),
166 error = %e,
167 "special-token merge from tokenizer_config.json failed; skipping"
168 );
169 return;
170 }
171 };
172 if added > 0 {
173 let promoted: Vec<&str> = to_add.iter().map(|t| t.content.as_str()).collect();
179 tracing::warn!(
180 target: "tokenizer",
181 path = %cfg_path.display(),
182 added,
183 candidates = to_add.len(),
184 promoted = ?promoted,
185 "merged additional special tokens from tokenizer_config.json"
186 );
187 }
188}
189
190impl Encoder for HuggingFaceTokenizer {
191 fn encode(&self, input: &str) -> Result<Encoding> {
192 let encoding = self
194 .tokenizer
195 .encode(input, self.options.add_special_tokens)
196 .map_err(|err| Error::msg(format!("Error tokenizing input: {err}")))?;
197
198 Ok(Encoding::Hf(Box::new(encoding)))
199 }
200
201 fn encode_segments(&self, segments: &[super::EncodeSegment<'_>]) -> Result<Encoding> {
202 let (trusted, ordinary) = self.segment_tokenizers.get_or_init(|| {
203 let mut trusted = self.tokenizer.clone();
204 trusted.with_padding(None);
207 trusted
208 .with_truncation(None)
209 .expect("removing truncation cannot fail");
210 trusted.with_post_processor(None::<tokenizers::processors::PostProcessorWrapper>);
211 let mut ordinary = trusted.clone();
212 ordinary.set_encode_special_tokens(true);
216 (trusted, ordinary)
217 });
218 let encodings = segments
219 .iter()
220 .map(|segment| {
221 let tokenizer = if segment.allow_special {
222 trusted
223 } else {
224 ordinary
225 };
226 tokenizer
227 .encode(segment.text, false)
228 .map_err(|error| Error::msg(format!("Error tokenizing segment: {error}")))
229 })
230 .collect::<Result<Vec<_>>>()?;
231 let merged = tokenizers::Encoding::merge(encodings, true);
232 let encoding = self
233 .tokenizer
234 .post_process(merged, None, self.options.add_special_tokens)
235 .map_err(|error| Error::msg(format!("Error post-processing segments: {error}")))?;
236 Ok(Encoding::Hf(Box::new(encoding)))
237 }
238
239 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
240 let hf_encodings = self
241 .tokenizer
242 .encode_batch(inputs.to_vec(), self.options.add_special_tokens)
243 .map_err(|err| Error::msg(format!("Error batch tokenizing input: {err}")))?;
244
245 let encodings = hf_encodings
246 .into_iter()
247 .map(|enc| Encoding::Hf(Box::new(enc)))
248 .collect();
249
250 Ok(encodings)
251 }
252}
253
254impl Decoder for HuggingFaceTokenizer {
255 fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
256 token_ids
257 .iter()
258 .rev()
259 .find(|id| !skip_special_tokens || !self.special_ids.contains(id))
260 .is_some_and(|id| self.byte_fallback_ids.contains(id))
261 }
262
263 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
264 let text = self
266 .tokenizer
267 .decode(token_ids, skip_special_tokens)
268 .map_err(|err| Error::msg(format!("Error de-tokenizing input: {err}")))?;
269
270 Ok(text.into())
271 }
272}
273
274impl Tokenizer for HuggingFaceTokenizer {
275 fn validate_prefix_cache(&self) -> Result<()> {
276 if self.options.add_special_tokens {
277 return Err(Error::msg(
278 "HuggingFace tokenizers configured with add_special_tokens=true must remain uncached",
279 ));
280 }
281 if self.tokenizer.get_truncation().is_some() || self.tokenizer.get_padding().is_some() {
284 return Err(Error::msg(
285 "HuggingFace tokenizers with truncation or padding must remain uncached",
286 ));
287 }
288 Ok(())
289 }
290
291 fn with_options(mut self, options: TokenizerOptions) -> Self {
294 self.options = options;
295 self
296 }
297
298 fn vocab_size(&self) -> Option<usize> {
299 Some(self.tokenizer.get_vocab_size(true))
300 }
301
302 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
303 Ok(self.tokenizer.token_to_id(token))
304 }
305
306 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
307 let mut ids: Vec<TokenIdType> = self
308 .tokenizer
309 .get_added_tokens_decoder()
310 .into_iter()
311 .filter_map(|(id, token)| token.special.then_some(id))
312 .collect();
313 ids.sort_unstable();
314 Ok(ids)
315 }
316
317 fn num_special_tokens_added(&self) -> Result<usize> {
318 Ok(self
319 .tokenizer
320 .get_post_processor()
321 .map_or(0, |processor| processor.added_tokens(false)))
322 }
323}
324
325impl From<HfTokenizer> for HuggingFaceTokenizer {
326 fn from(tokenizer: HfTokenizer) -> Self {
327 Self::from_tokenizer(tokenizer)
328 }
329}
330
331#[cfg(test)]
332mod byte_fallback_stream_tests {
333 use super::*;
334
335 fn tokenizer() -> super::super::Tokenizer {
336 let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
337 "version": "1.0", "truncation": null, "padding": null,
338 "added_tokens": [{"id": 6, "content": "<eos>", "special": true,
339 "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}],
340 "normalizer": null, "pre_tokenizer": null, "post_processor": null,
341 "decoder": {"type": "Sequence", "decoders": [
342 {"type": "ByteFallback"}, {"type": "Fuse"}]},
343 "model": {"type": "BPE", "vocab": {
344 "<0x61>": 0, "<0xF5>": 1, "<0xC3>": 2, "<0xA9>": 3,
345 " hello": 4, "!": 5, "<eos>": 6}, "merges": [], "byte_fallback": true}
346 }))
347 .unwrap();
348 std::sync::Arc::new(HuggingFaceTokenizer::from_tokenizer(hf)).into()
349 }
350
351 #[test]
352 fn byte_runs_match_full_decode() {
353 let tokenizer = tokenizer();
354 for (ids, skip) in [
358 (vec![0, 1, 4], false),
359 (vec![2, 3, 1, 4], false),
360 (vec![2, 3, 5], false),
361 (vec![0], false),
362 (vec![2, 3], false),
363 (vec![2], false),
364 (vec![0, 1], false),
365 (vec![0, 6, 1, 4], false),
366 (vec![0, 6, 1, 4], true),
367 ] {
368 let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
369 let mut stream = tokenizer.decode_stream(&[], skip);
370 let mut actual = String::new();
371 for id in &ids {
372 actual.push_str(&stream.step(*id).unwrap().unwrap_or_default());
373 }
374 actual.push_str(&stream.finish().unwrap().unwrap_or_default());
375 assert_eq!(actual, expected, "ids={ids:?}, skip={skip}");
376 assert_eq!(stream.finish().unwrap(), None);
377 }
378 }
379
380 #[test]
381 fn only_byte_suffix_is_delayed_and_prompt_is_not_emitted() {
382 let tokenizer = tokenizer();
383 let mut stream = tokenizer.decode_stream(&[4], true);
384 assert_eq!(stream.step(5).unwrap().as_deref(), Some("!"));
385 assert_eq!(stream.step(0).unwrap(), None);
386 assert_eq!(stream.step(1).unwrap(), None);
387 assert_eq!(stream.step(4).unwrap().as_deref(), Some("�� hello"));
388 assert_eq!(stream.finish().unwrap(), None);
389 }
390}
391
392#[cfg(test)]
393mod tests {
394 use super::*;
403 use std::fs;
404 use tempfile::TempDir;
405
406 #[test]
407 fn merge_gate_round_trips_through_decode() {
408 const TOKENIZER_JSON: &str = r#"{
414 "version": "1.0",
415 "truncation": null,
416 "padding": null,
417 "added_tokens": [
418 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
419 ],
420 "normalizer": null,
421 "pre_tokenizer": null,
422 "post_processor": null,
423 "decoder": null,
424 "model": {
425 "type": "WordLevel",
426 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<|special_kept|>": 3, "<|special_dropped|>": 4},
427 "unk_token": "<unk>"
428 }
429 }"#;
430
431 const TOKENIZER_CONFIG_JSON: &str = r#"{
440 "added_tokens_decoder": {
441 "3": {"content": "<|special_kept|>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
442 "4": {"content": "<|special_dropped|>", "special": false, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
443 }
444 }"#;
445
446 let dir = TempDir::new().unwrap();
447 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
448 fs::write(
449 dir.path().join("tokenizer_config.json"),
450 TOKENIZER_CONFIG_JSON,
451 )
452 .unwrap();
453
454 let mut tokenizer = HfTokenizer::from_file(dir.path().join("tokenizer.json")).unwrap();
455 merge_special_tokens_from_config(&mut tokenizer, dir.path());
456
457 let specials: Vec<String> = {
459 let mut v: Vec<String> = tokenizer
460 .get_added_tokens_decoder()
461 .values()
462 .filter(|t| t.special)
463 .map(|t| t.content.clone())
464 .collect();
465 v.sort();
466 v
467 };
468 assert_eq!(
469 specials,
470 vec!["<unk>".to_string(), "<|special_kept|>".to_string()],
471 "<|special_kept|> promoted; <|special_dropped|> stayed non-special"
472 );
473
474 let enc_kept = tokenizer.encode("<|special_kept|>", false).unwrap();
478 let decoded_strip = tokenizer.decode(enc_kept.get_ids(), true).unwrap();
479 assert!(
480 !decoded_strip.contains("<|special_kept|>"),
481 "promoted special:true token must be stripped under skip_special_tokens=true; got {decoded_strip:?}"
482 );
483
484 let enc_drop = tokenizer.encode("<|special_dropped|>", false).unwrap();
485 let decoded_keep = tokenizer.decode(enc_drop.get_ids(), true).unwrap();
486 assert!(
487 decoded_keep.contains("<|special_dropped|>"),
488 "non-promoted special:false token must survive skip_special_tokens=true; got {decoded_keep:?}"
489 );
490 }
491
492 #[test]
493 fn add_special_tokens_flag_controls_encode() {
494 const TOKENIZER_JSON: &str = r#"{
499 "version": "1.0",
500 "truncation": null,
501 "padding": null,
502 "added_tokens": [
503 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
504 {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
505 ],
506 "normalizer": null,
507 "pre_tokenizer": null,
508 "post_processor": {
509 "type": "TemplateProcessing",
510 "single": [
511 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
512 {"Sequence": {"id": "A", "type_id": 0}}
513 ],
514 "pair": [
515 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
516 {"Sequence": {"id": "A", "type_id": 0}},
517 {"Sequence": {"id": "B", "type_id": 0}}
518 ],
519 "special_tokens": {
520 "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
521 }
522 },
523 "decoder": null,
524 "model": {
525 "type": "WordLevel",
526 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
527 "unk_token": "<unk>"
528 }
529 }"#;
530
531 let dir = TempDir::new().unwrap();
532 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
533 let path = dir.path().join("tokenizer.json");
534 let path = path.to_str().unwrap();
535
536 let ids = |enc: &Encoding| match enc {
537 Encoding::Hf(e) => e.get_ids().to_vec(),
538 _ => panic!("expected Hf encoding"),
539 };
540
541 let plain = HuggingFaceTokenizer::from_file(path).unwrap();
543 assert_eq!(ids(&plain.encode("hello").unwrap()), vec![1]);
544
545 let with_bos =
546 HuggingFaceTokenizer::from_file(path)
547 .unwrap()
548 .with_options(TokenizerOptions {
549 add_special_tokens: true,
550 });
551 assert_eq!(ids(&with_bos.encode("hello").unwrap()), vec![3, 1]);
552 let batch = with_bos.encode_batch(&["hello", "world"]).unwrap();
553 assert_eq!(ids(&batch[0]), vec![3, 1]);
554 assert_eq!(ids(&batch[1]), vec![3, 2]);
555
556 use crate::Tokenizer as TokenizerWrapper;
560 let wrapper_plain = TokenizerWrapper::from_file(path).unwrap();
561 assert_eq!(ids(&wrapper_plain.encode("hello").unwrap()), vec![1]);
562
563 let wrapper_bos = TokenizerWrapper::from_file_with_options(
564 path,
565 TokenizerOptions {
566 add_special_tokens: true,
567 },
568 )
569 .unwrap();
570 assert_eq!(ids(&wrapper_bos.encode("hello").unwrap()), vec![3, 1]);
571 }
572
573 #[test]
574 fn truncation_or_padding_blocks_prefix_cache() {
575 let path = concat!(
576 env!("CARGO_MANIFEST_DIR"),
577 "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
578 );
579 let mut truncated = HfTokenizer::from_file(path).unwrap();
580 truncated
581 .with_truncation(Some(tokenizers::TruncationParams::default()))
582 .unwrap();
583 let mut padded = HfTokenizer::from_file(path).unwrap();
584 padded.with_padding(Some(tokenizers::PaddingParams::default()));
585 for hf in [truncated, padded] {
586 assert!(
587 HuggingFaceTokenizer::from_tokenizer(hf)
588 .validate_prefix_cache()
589 .is_err()
590 );
591 }
592 }
593
594 #[test]
595 fn vocab_introspection_accessors() {
596 const TOKENIZER_JSON: &str = r#"{
597 "version": "1.0",
598 "truncation": null,
599 "padding": null,
600 "added_tokens": [
601 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
602 ],
603 "normalizer": null,
604 "pre_tokenizer": null,
605 "post_processor": null,
606 "decoder": null,
607 "model": {
608 "type": "WordLevel",
609 "vocab": {"<unk>": 0, "hello": 1, "world": 2},
610 "unk_token": "<unk>"
611 }
612 }"#;
613
614 let dir = TempDir::new().unwrap();
615 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
616 let path = dir.path().join("tokenizer.json");
617
618 let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
619 assert_eq!(tokenizer.vocab_size(), Some(3));
620 assert_eq!(tokenizer.token_to_id("hello").unwrap(), Some(1));
621 assert_eq!(tokenizer.special_token_ids().unwrap(), vec![0]);
622 assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 0);
623 }
624
625 #[test]
626 fn num_special_tokens_added_reflects_post_processor_additions() {
627 const TOKENIZER_JSON: &str = r#"{
628 "version": "1.0",
629 "truncation": null,
630 "padding": null,
631 "added_tokens": [
632 {"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
633 {"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
634 ],
635 "normalizer": null,
636 "pre_tokenizer": null,
637 "post_processor": {
638 "type": "TemplateProcessing",
639 "single": [
640 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
641 {"Sequence": {"id": "A", "type_id": 0}}
642 ],
643 "pair": [
644 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
645 {"Sequence": {"id": "A", "type_id": 0}},
646 {"Sequence": {"id": "B", "type_id": 0}}
647 ],
648 "special_tokens": {
649 "<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
650 }
651 },
652 "decoder": null,
653 "model": {
654 "type": "WordLevel",
655 "vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
656 "unk_token": "<unk>"
657 }
658 }"#;
659
660 let dir = TempDir::new().unwrap();
661 fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
662 let path = dir.path().join("tokenizer.json");
663
664 let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
665 assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 1);
666 }
667}
668
669#[cfg(test)]
670mod segmented_tests {
671 use super::*;
672 use crate::EncodeSegment;
673
674 #[test]
675 fn segmented_hf_preserves_literal_controls_and_adds_bos_once() {
676 let mut vocab = serde_json::json!({"<unk>":0,"<|call|>":1,"<bos>":2});
677 for ch in "<|cal>".chars() {
678 let id = vocab.as_object().unwrap().len();
679 vocab[ch.to_string()] = serde_json::json!(id);
680 }
681 let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
682 "version":"1.0", "truncation":null, "padding":null,
683 "normalizer":null, "pre_tokenizer":null, "decoder":{"type":"Fuse"},
684 "added_tokens":[{"id":1,"content":"<|call|>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false},
685 {"id":2,"content":"<bos>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
686 "post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<bos>","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],
687 "pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"<bos>":{"id":"<bos>","ids":[2],"tokens":["<bos>"]}}},
688 "model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
689 })).unwrap();
690 for add_special_tokens in [false, true] {
691 let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf.clone())
692 .with_options(TokenizerOptions { add_special_tokens });
693 let encoding = tokenizer
694 .encode_segments(&[
695 EncodeSegment::control("<|call|>"),
696 EncodeSegment::ordinary("<|call|>"),
697 ])
698 .unwrap();
699 let ids = encoding.token_ids();
700 assert_eq!(ids.iter().filter(|id| **id == 1).count(), 1);
701 assert_eq!(
702 ids.iter().filter(|id| **id == 2).count(),
703 usize::from(add_special_tokens)
704 );
705 let decoded: String = tokenizer.decode(ids, true).unwrap().into();
706 assert_eq!(decoded, "<|call|>");
707 assert_eq!(
708 tokenizer
709 .encode("<|call|>")
710 .unwrap()
711 .token_ids()
712 .iter()
713 .filter(|id| **id == 1)
714 .count(),
715 1
716 );
717 }
718 }
719}