1pub mod added_tokens;
2pub mod decoders;
3pub mod json_structs;
4pub mod models;
5pub mod normalizers;
6pub mod post_processors;
7pub mod pre_tokenized;
8pub mod pre_tokenizers;
9mod tkz;
10
11use std::{borrow::Cow, fs, path::Path};
12
13use rayon::prelude::*;
14use serde_json::Value;
15
16const WIDE_BATCH_TASKS: usize = 4;
18const SHORT_BATCH_INPUT_BYTES: usize = 16 * 1024;
20const PARALLEL_BATCH_BYTES_PER_THREAD: usize = 2 * 1024;
22const PARALLEL_CACHE_BATCH_BYTES: usize = 64 * 1024;
24const COMPACT_RAGGED_OUTPUT_BYTES: usize = 8 * 1024 * 1024;
26const RAGGED_CHUNKS_PER_WORKER: usize = 8;
28const SHORT_OUTPUT_BYTES: usize = 16 * 1024;
30
31fn output_capacity(input_bytes: usize) -> usize {
33 if input_bytes <= SHORT_OUTPUT_BYTES {
34 return input_bytes.div_ceil(2).saturating_add(16).min(input_bytes);
35 }
36 input_bytes.div_ceil(3)
37}
38
39pub use self::{
40 added_tokens::{AddedTokenInfo, AddedTokens},
41 json_structs::{
42 AddedTokenConfig, DecoderConfig, ModelConfig, NormalizerConfig, PostProcessorConfig,
43 PreTokenizerConfig, TokenizerJson,
44 },
45 models::Model,
46 normalizers::{Nfc, Normalizer, Replace},
47 post_processors::PostProcessor,
48 pre_tokenizers::{ByteLevel, PreTokenizer, Split, SplitBehavior},
49};
50
51use self::{
52 added_tokens::Segment,
53 decoders::Decoder,
54 pre_tokenized::{PreTokenizedString, Split as PtSplit},
55 pre_tokenizers::FusedSplits,
56};
57
58#[derive(Debug, thiserror::Error)]
60pub enum Error {
61 #[error("failed to read tokenizer files: {0}")]
62 Io(#[from] std::io::Error),
63
64 #[error("failed to parse tokenizer files: {0}")]
65 Json(#[from] serde_json::Error),
66
67 #[error("invalid .tkz tokenizer: {0}")]
68 Tkz(String),
69
70 #[error("normalizer error: {0}")]
71 Normalizer(#[from] normalizers::Error),
72
73 #[error("pre-tokenizer error: {0}")]
74 PreTokenizer(#[from] pre_tokenizers::Error),
75
76 #[error("post-processor error: {0}")]
77 PostProcessor(#[from] post_processors::Error),
78
79 #[error("decoder error: {0}")]
80 Decoder(#[from] decoders::Error),
81
82 #[error("model error: {0}")]
83 Model(String),
84}
85
86pub struct Tokenizer {
88 added_tokens: Option<AddedTokens>,
89 normalizer: Option<Normalizer>,
90 pre_tokenizer: Option<PreTokenizer>,
91 model: Model,
92 post_processor: Option<PostProcessor>,
93 decoder: Option<Decoder>,
94 needs_vocab_splitting: bool,
100}
101
102impl Tokenizer {
103 fn build(json: TokenizerJson) -> Result<Self, Error> {
105 let normalizer = json.normalizer.map(Normalizer::from_config).transpose()?;
106 let added_tokens =
107 AddedTokens::from_configs_with_normalizer(&json.added_tokens, normalizer.as_ref())
108 .map_err(Error::Model)?;
109 let pre_tokenizer = json
110 .pre_tokenizer
111 .map(PreTokenizer::from_config)
112 .transpose()?;
113 let model = Model::from_config(json.model).map_err(Error::Model)?;
114 let post_processor = json
115 .post_processor
116 .map(PostProcessor::from_config)
117 .transpose()?;
118 let decoder = json.decoder.map(Decoder::from_config).transpose()?;
119
120 let needs_vocab_splitting = !pre_tokenizer
124 .as_ref()
125 .is_some_and(PreTokenizer::contains_byte_level);
126
127 Ok(Self {
128 added_tokens,
129 normalizer,
130 pre_tokenizer,
131 model,
132 post_processor,
133 decoder,
134 needs_vocab_splitting,
135 })
136 }
137
138 pub fn from_json(json: Value) -> Result<Self, Error> {
140 let json: TokenizerJson = serde_json::from_value(json)?;
141 Self::build(json)
142 }
143
144 pub fn from_file(path: &Path) -> Result<Self, Error> {
146 let json: TokenizerJson = serde_json::from_str(&fs::read_to_string(path)?)?;
147 Self::build(json)
148 }
149
150 pub fn from_file_with_tkz_cache(path: &Path, enabled: bool) -> Result<Self, Error> {
156 if !enabled {
157 return Self::from_file(path);
158 }
159 tkz::load_or_create(path)
160 }
161
162 pub fn normalizer(&self) -> Option<&Normalizer> {
164 self.normalizer.as_ref()
165 }
166
167 pub fn pre_tokenizer(&self) -> Option<&PreTokenizer> {
169 self.pre_tokenizer.as_ref()
170 }
171
172 pub fn post_processor(&self) -> Option<&PostProcessor> {
174 self.post_processor.as_ref()
175 }
176
177 pub fn model(&self) -> &Model {
179 &self.model
180 }
181
182 pub fn added_tokens(&self) -> Option<&AddedTokens> {
184 self.added_tokens.as_ref()
185 }
186
187 pub fn decoder(&self) -> Option<&Decoder> {
189 self.decoder.as_ref()
190 }
191
192 pub fn encode(&self, input: &str) -> Result<Vec<u32>, Error> {
197 self.encode_with_special_tokens(input, false)
198 }
199
200 pub fn encode_with_special_tokens(
205 &self,
206 input: &str,
207 add_special_tokens: bool,
208 ) -> Result<Vec<u32>, Error> {
209 self.encode_with_cache_mode(input, add_special_tokens, false)
210 }
211
212 fn encode_with_cache_mode(
214 &self,
215 input: &str,
216 add_special_tokens: bool,
217 use_parallel_cache: bool,
218 ) -> Result<Vec<u32>, Error> {
219 if input.is_empty() {
220 return if add_special_tokens {
221 Ok(self.post_process(Vec::new(), true))
222 } else {
223 Ok(Vec::new())
224 };
225 }
226
227 let fused_byte_level = self
228 .pre_tokenizer
229 .as_ref()
230 .and_then(PreTokenizer::fused_byte_level);
231 let fused_split = fused_byte_level.and_then(|(splits, _)| splits?.single());
232
233 if let Some((None, byte_level)) = fused_byte_level
236 && self.normalizer.is_none()
237 {
238 let mut ids = Vec::with_capacity(output_capacity(input.len()));
239 self.encode_fused_byte_level_into(input, byte_level, &mut ids, use_parallel_cache)?;
240 return Ok(self.post_process(ids, add_special_tokens));
241 }
242
243 if let Some((Some(splits), _)) = fused_byte_level {
244 let mut ids = Vec::new();
245 self.encode_fused_split_into(input, splits, &mut ids, use_parallel_cache)?;
246 return Ok(self.post_process(ids, add_special_tokens));
247 }
248
249 let (mut pts, split_applied) = self.build_pre_tokenized_for_encode(input, fused_split);
251
252 if let Some((split, byte_level)) = fused_byte_level {
254 if let Some(split) = split
255 && !split_applied
256 {
257 split.pre_tokenize(&mut pts)?;
258 }
259 byte_level.pre_tokenize_fused(&mut pts);
260 let ids = pts
261 .tokenize_batched(|buf, splits, out| {
262 self.model.tokenize_batch_fused(buf, splits, out)
263 })
264 .map_err(Error::Model)?;
265 return Ok(self.post_process(ids, add_special_tokens));
266 }
267
268 if let Some(ref pt) = self.pre_tokenizer {
270 pt.pre_tokenize(&mut pts)?;
271 }
272
273 if self.needs_vocab_splitting
280 && let Some(table) = self.model.bigram_bridge_table()
281 {
282 split_on_unbridgeable_bigrams(&mut pts, table);
283 }
284
285 let ids = pts
287 .tokenize(|text, out| self.model.tokenize_into(text, out))
288 .map_err(Error::Model)?;
289
290 Ok(self.post_process(ids, add_special_tokens))
292 }
293
294 pub fn encode_batch<S: AsRef<str> + Sync>(
296 &self,
297 inputs: &[S],
298 add_special_tokens: bool,
299 ) -> Result<Vec<Vec<u32>>, Error> {
300 let total_bytes = inputs
301 .iter()
302 .map(|input| input.as_ref().len())
303 .sum::<usize>();
304 let parallel_bytes =
305 PARALLEL_BATCH_BYTES_PER_THREAD.saturating_mul(rayon::current_num_threads());
306 if total_bytes <= parallel_bytes
308 && (inputs.len() < WIDE_BATCH_TASKS || !self.needs_vocab_splitting)
309 {
310 return inputs
311 .iter()
312 .map(|input| self.encode_with_special_tokens(input.as_ref(), add_special_tokens))
313 .collect();
314 }
315
316 let outer_tasks = inputs.len().min(rayon::current_num_threads());
317 if outer_tasks < 2 {
318 return inputs
319 .par_iter()
320 .map(|input| self.encode_with_special_tokens(input.as_ref(), add_special_tokens))
321 .collect();
322 }
323
324 self.encode_outer_batch(inputs, add_special_tokens, outer_tasks)
325 }
326
327 pub fn encode_batch_ragged<S: AsRef<str> + Sync>(
329 &self,
330 inputs: &[S],
331 add_special_tokens: bool,
332 ) -> Result<(Vec<u32>, Vec<usize>), Error> {
333 let fused_byte_level = self
334 .pre_tokenizer
335 .as_ref()
336 .and_then(PreTokenizer::fused_byte_level);
337 let total_bytes = inputs
338 .iter()
339 .map(|input| input.as_ref().len())
340 .sum::<usize>();
341 let parallel_bytes =
342 PARALLEL_BATCH_BYTES_PER_THREAD.saturating_mul(rayon::current_num_threads());
343 if !add_special_tokens && let Some((Some(splits), _)) = fused_byte_level {
344 return self.encode_fused_split_ragged(inputs, splits, total_bytes, parallel_bytes);
345 }
346
347 if !add_special_tokens
348 && (inputs.len() == 1 || total_bytes <= parallel_bytes)
349 && self.normalizer.is_none()
350 && let Some((None, byte_level)) = fused_byte_level
351 {
352 return self.encode_fused_ragged_serial(inputs, byte_level, total_bytes);
353 }
354
355 if !add_special_tokens
356 && inputs.len() >= 2
357 && self.normalizer.is_none()
358 && let Some((None, byte_level)) = fused_byte_level
359 {
360 return self.encode_fused_ragged_parallel(inputs, byte_level, total_bytes);
361 }
362
363 let rows = self.encode_batch(inputs, add_special_tokens)?;
364 let total = rows.iter().map(Vec::len).sum();
365 let mut ids = Vec::with_capacity(total);
366 let mut lengths = Vec::with_capacity(rows.len());
367 for row in rows {
368 lengths.push(row.len());
369 ids.extend(row);
370 }
371 Ok((ids, lengths))
372 }
373
374 #[inline(never)]
376 fn encode_fused_ragged_serial<S: AsRef<str> + Sync>(
377 &self,
378 inputs: &[S],
379 byte_level: &ByteLevel,
380 total_bytes: usize,
381 ) -> Result<(Vec<u32>, Vec<usize>), Error> {
382 let mut ids = Vec::with_capacity(output_capacity(total_bytes));
383 let mut lengths = Vec::with_capacity(inputs.len());
384 if self.added_tokens.as_ref().is_none_or(|added_tokens| {
385 inputs
386 .iter()
387 .all(|input| !added_tokens.has_candidate(input.as_ref()))
388 }) {
389 self.model
390 .tokenize_fused_stream("", &mut ids, false, |stream| {
391 let mut start = 0;
392 for input in inputs {
393 let input = input.as_ref();
394 byte_level.stream_fused(input, stream);
395 let end = stream.output_len();
396 lengths.push(end - start);
397 start = end;
398 }
399 })
400 .map_err(Error::Model)?;
401 return Ok((ids, lengths));
402 }
403 for input in inputs {
404 let start = ids.len();
405 self.encode_fused_byte_level_into(input.as_ref(), byte_level, &mut ids, false)?;
406 lengths.push(ids.len() - start);
407 }
408 Ok((ids, lengths))
409 }
410
411 fn encode_fused_split_ragged<S: AsRef<str> + Sync>(
413 &self,
414 inputs: &[S],
415 splits: FusedSplits<'_>,
416 total_bytes: usize,
417 parallel_bytes: usize,
418 ) -> Result<(Vec<u32>, Vec<usize>), Error> {
419 let encode_chunk = |chunk: &[S]| {
420 let chunk_bytes = chunk
421 .iter()
422 .map(|input| input.as_ref().len())
423 .sum::<usize>();
424 let capacity = if total_bytes >= COMPACT_RAGGED_OUTPUT_BYTES {
425 chunk_bytes.div_ceil(2).saturating_add(16)
426 } else {
427 chunk_bytes.saturating_add(3)
428 };
429 let mut ids = Vec::with_capacity(capacity);
430 let mut lengths = Vec::with_capacity(chunk.len());
431 let use_parallel_cache = inputs.len() > 2 && total_bytes >= PARALLEL_CACHE_BATCH_BYTES;
432 if chunk
433 .iter()
434 .all(|input| self.can_encode_fused_split(input.as_ref()))
435 {
436 self.model
437 .tokenize_fused_stream("", &mut ids, use_parallel_cache, |stream| {
438 let mut start = 0;
439 for input in chunk {
440 let input = input.as_ref();
441 let normalized = self
442 .normalizer
443 .as_ref()
444 .map_or(Cow::Borrowed(input), |normalizer| {
445 normalizer.normalize(input)
446 });
447 let input = normalized.as_ref();
448 splits.stream_into(input, stream);
449 let end = stream.output_len();
450 lengths.push(end - start);
451 start = end;
452 }
453 })
454 .map_err(Error::Model)?;
455 return Ok((ids, lengths));
456 }
457 for input in chunk {
458 let input = input.as_ref();
459 let start = ids.len();
460 self.encode_fused_split_into(input, splits, &mut ids, use_parallel_cache)?;
461 lengths.push(ids.len() - start);
462 }
463 Ok((ids, lengths))
464 };
465
466 if inputs.len() == 1
467 && total_bytes >= PARALLEL_CACHE_BATCH_BYTES
468 && self.normalizer.is_none()
469 {
470 let input = inputs[0].as_ref();
471 let workers = rayon::current_num_threads().min(WIDE_BATCH_TASKS);
472 if let Some(ranges) = splits.newline_partition_ranges(input, workers)
473 && ranges.len() > 1
474 {
475 let chunks = ranges
476 .into_par_iter()
477 .map(|range| {
478 let input = &input[range];
479 let mut ids = Vec::with_capacity(output_capacity(input.len()));
480 self.model
481 .tokenize_fused_stream("", &mut ids, true, |stream| {
482 splits.stream_into(input, stream);
483 })
484 .map_err(Error::Model)?;
485 Ok(ids)
486 })
487 .collect::<Result<Vec<_>, Error>>()?;
488 let token_count = chunks.iter().map(Vec::len).sum();
489 let mut ids = Vec::with_capacity(token_count);
490 for chunk in chunks {
491 ids.extend(chunk);
492 }
493 return Ok((ids, vec![token_count]));
494 }
495 }
496
497 if inputs.len() < 3 || total_bytes <= parallel_bytes {
499 return encode_chunk(inputs);
500 }
501
502 let workers = inputs.len().min(rayon::current_num_threads());
503 let chunk_size = inputs.len().div_ceil(workers * RAGGED_CHUNKS_PER_WORKER);
504 let chunks = inputs
505 .par_chunks(chunk_size)
506 .map(encode_chunk)
507 .collect::<Result<Vec<_>, Error>>()?;
508 let token_count = chunks.iter().map(|(ids, _)| ids.len()).sum();
509 let mut ids = Vec::with_capacity(token_count);
510 let mut lengths = Vec::with_capacity(inputs.len());
511 for (chunk_ids, chunk_lengths) in chunks {
512 ids.extend(chunk_ids);
513 lengths.extend(chunk_lengths);
514 }
515 Ok((ids, lengths))
516 }
517
518 #[inline(never)]
520 fn encode_fused_ragged_parallel<S: AsRef<str> + Sync>(
521 &self,
522 inputs: &[S],
523 byte_level: &ByteLevel,
524 total_bytes: usize,
525 ) -> Result<(Vec<u32>, Vec<usize>), Error> {
526 let workers = inputs.len().min(rayon::current_num_threads());
527 let chunk_size = inputs.len().div_ceil(workers * RAGGED_CHUNKS_PER_WORKER);
528 let use_parallel_cache = total_bytes >= PARALLEL_CACHE_BATCH_BYTES;
529 let chunks = inputs
530 .par_chunks(chunk_size)
531 .map(|chunk| {
532 let chunk_bytes = chunk
533 .iter()
534 .map(|input| input.as_ref().len())
535 .sum::<usize>();
536 let mut ids = Vec::with_capacity(output_capacity(chunk_bytes));
537 let mut lengths = Vec::with_capacity(chunk.len());
538 for input in chunk {
539 let start = ids.len();
540 self.encode_fused_byte_level_into(
541 input.as_ref(),
542 byte_level,
543 &mut ids,
544 use_parallel_cache,
545 )?;
546 lengths.push(ids.len() - start);
547 }
548 Ok((ids, lengths))
549 })
550 .collect::<Result<Vec<_>, Error>>()?;
551
552 let token_count = chunks.iter().map(|(ids, _)| ids.len()).sum();
553 let mut ids = Vec::with_capacity(token_count);
554 let mut lengths = Vec::with_capacity(inputs.len());
555 for (chunk_ids, chunk_lengths) in chunks {
556 ids.extend(chunk_ids);
557 lengths.extend(chunk_lengths);
558 }
559 Ok((ids, lengths))
560 }
561
562 #[inline(never)]
564 fn encode_outer_batch<S: AsRef<str> + Sync>(
565 &self,
566 inputs: &[S],
567 add_special_tokens: bool,
568 outer_tasks: usize,
569 ) -> Result<Vec<Vec<u32>>, Error> {
570 let use_parallel_cache = inputs
571 .iter()
572 .map(|input| input.as_ref().len())
573 .sum::<usize>()
574 >= PARALLEL_CACHE_BATCH_BYTES;
575 inputs
576 .par_iter()
577 .map(|input| {
578 let input = input.as_ref();
579 if outer_tasks >= WIDE_BATCH_TASKS || input.len() <= SHORT_BATCH_INPUT_BYTES {
582 pre_tokenized::without_inner_parallelism(|| {
583 self.encode_with_cache_mode(input, add_special_tokens, use_parallel_cache)
584 })
585 } else {
586 self.encode_with_cache_mode(input, add_special_tokens, use_parallel_cache)
587 }
588 })
589 .collect()
590 }
591
592 fn encode_fused_byte_level_into(
594 &self,
595 input: &str,
596 byte_level: &ByteLevel,
597 ids: &mut Vec<u32>,
598 use_parallel_cache: bool,
599 ) -> Result<(), Error> {
600 if let Some(added_tokens) = &self.added_tokens
601 && added_tokens.has_candidate(input)
602 {
603 for segment in added_tokens.split(input) {
604 match segment {
605 Segment::Token(id) => ids.push(id),
606 Segment::Text(text) => self
607 .model
608 .tokenize_fused_stream(text, ids, use_parallel_cache, |stream| {
609 byte_level.stream_fused(text, stream)
610 })
611 .map_err(Error::Model)?,
612 }
613 }
614 return Ok(());
615 }
616
617 self.model
618 .tokenize_fused_stream(input, ids, use_parallel_cache, |stream| {
619 byte_level.stream_fused(input, stream)
620 })
621 .map_err(Error::Model)
622 }
623
624 fn encode_fused_split_into(
626 &self,
627 input: &str,
628 splits: FusedSplits<'_>,
629 ids: &mut Vec<u32>,
630 use_parallel_cache: bool,
631 ) -> Result<(), Error> {
632 if !self.can_encode_fused_split(input) {
633 let segments = self.segment_input(input);
634 let normalized_added_tokens = self
635 .added_tokens
636 .as_ref()
637 .filter(|added_tokens| added_tokens.has_normalized());
638 self.model
639 .tokenize_fused_stream(input, ids, use_parallel_cache, |stream| {
640 for segment in segments {
641 match segment {
642 Segment::Token(id) => stream.push_id(id),
643 Segment::Text(text) => {
644 let normalized = self
645 .normalizer
646 .as_ref()
647 .map_or(Cow::Borrowed(text), |normalizer| {
648 normalizer.normalize(text)
649 });
650 if let Some(added_tokens) = normalized_added_tokens {
651 for segment in
652 added_tokens.split_normalized(normalized.as_ref())
653 {
654 match segment {
655 Segment::Token(id) => stream.push_id(id),
656 Segment::Text(text) => {
657 splits.stream_into(text, stream);
658 stream.flush_pending();
659 }
660 }
661 }
662 } else {
663 let text = normalized.as_ref();
664 splits.stream_into(text, stream);
665 stream.flush_pending();
666 }
667 }
668 }
669 }
670 })
671 .map_err(Error::Model)?;
672 return Ok(());
673 }
674
675 let normalized = self
676 .normalizer
677 .as_ref()
678 .map_or(Cow::Borrowed(input), |normalizer| {
679 normalizer.normalize(input)
680 });
681 let input = normalized.as_ref();
682 self.model
683 .tokenize_fused_stream(input, ids, use_parallel_cache, |stream| {
684 splits.stream_into(input, stream)
685 })
686 .map_err(Error::Model)?;
687 Ok(())
688 }
689
690 fn can_encode_fused_split(&self, input: &str) -> bool {
692 !self
693 .added_tokens
694 .as_ref()
695 .is_some_and(|added_tokens| added_tokens.has_candidate(input))
696 }
697
698 pub fn set_post_processor(&mut self, pp: Option<PostProcessor>) {
700 self.post_processor = pp;
701 }
702
703 pub fn set_normalizer(&mut self, normalizer: Option<Normalizer>) {
705 self.normalizer = normalizer;
706 }
707
708 pub fn post_process(&self, ids: Vec<u32>, add_special_tokens: bool) -> Vec<u32> {
709 match &self.post_processor {
710 Some(pp) => pp.post_process_single(ids, add_special_tokens),
711 None => ids,
712 }
713 }
714
715 pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
722 let mut tokens = Vec::with_capacity(ids.len());
723 for &id in ids {
724 if skip_special_tokens
725 && let Some(ref at) = self.added_tokens
726 && at.is_special(id)
727 {
728 continue;
729 }
730 if let Some(token_str) = self.id_to_token(id) {
735 tokens.push(token_str.to_string());
736 }
737 }
738
739 self.decode_tokens(tokens)
740 }
741
742 pub fn decode_tokens(&self, tokens: Vec<String>) -> Result<String, Error> {
748 match &self.decoder {
749 Some(dec) => dec.decode(tokens).map_err(Error::Decoder),
750 None => Ok(tokens.concat()),
751 }
752 }
753
754 pub fn decode_batch(
756 &self,
757 sentences: &[&[u32]],
758 skip_special_tokens: bool,
759 ) -> Result<Vec<String>, Error> {
760 sentences
761 .iter()
762 .map(|ids| self.decode(ids, skip_special_tokens))
763 .collect()
764 }
765
766 pub fn id_to_token(&self, id: u32) -> Option<&str> {
771 if let Some(ref at) = self.added_tokens
772 && let Some(s) = at.id_to_token(id)
773 {
774 return Some(s);
775 }
776 self.model.id_to_token(id)
777 }
778
779 pub fn token_to_id(&self, token: &str) -> Option<u32> {
784 if let Some(ref at) = self.added_tokens
785 && let Some(id) = at.token_to_id(token)
786 {
787 return Some(id);
788 }
789 self.model.token_to_id(token)
790 }
791
792 pub fn vocab_size(&self) -> usize {
794 let model_size = self.model.vocab_size();
795 let added_size = self.added_tokens.as_ref().map_or(0, |at| at.len());
796 model_size + added_size
797 }
798
799 pub fn is_special_token(&self, id: u32) -> bool {
801 self.added_tokens
802 .as_ref()
803 .is_some_and(|added_tokens| added_tokens.is_special(id))
804 }
805
806 pub fn build_pre_tokenized(&self, input: &str) -> PreTokenizedString {
811 let segments = self.segment_input(input);
812 self.build_pre_tokenized_from_segments(input, &segments)
813 }
814
815 fn segment_input<'a>(&self, input: &'a str) -> Vec<Segment<'a>> {
817 match &self.added_tokens {
818 Some(at) => at.split_non_normalized(input),
819 None => vec![Segment::Text(input)],
820 }
821 }
822
823 fn build_pre_tokenized_for_encode(
825 &self,
826 input: &str,
827 fused_split: Option<&Split>,
828 ) -> (PreTokenizedString, bool) {
829 let segments = self.segment_input(input);
830 if matches!(self.normalizer, Some(Normalizer::Nfc(_)))
831 && !self
832 .added_tokens
833 .as_ref()
834 .is_some_and(AddedTokens::has_normalized)
835 && let Some(split) = fused_split
836 && split.supports_ascii_nfc_fusion()
837 {
838 let capacity = input.len().div_ceil(4).max(segments.len());
839 let mut splits = Vec::with_capacity(capacity);
840 let mut buffer_len = 0;
841 let mut is_ascii = true;
842
843 for segment in &segments {
844 match segment {
845 Segment::Token(id) => splits.push(PtSplit {
846 range: buffer_len..buffer_len,
847 token_id: Some(*id),
848 }),
849 Segment::Text(text) => {
850 if !split.append_ascii_splits(text, buffer_len, &mut splits) {
851 is_ascii = false;
852 break;
853 }
854 buffer_len += text.len();
855 }
856 }
857 }
858
859 if is_ascii {
860 let mut buffer = String::with_capacity(buffer_len);
861 for segment in &segments {
862 if let Segment::Text(text) = segment {
863 buffer.push_str(text);
864 }
865 }
866 return (PreTokenizedString::new(buffer, splits), true);
867 }
868 }
869
870 (
871 self.build_pre_tokenized_from_segments(input, &segments),
872 false,
873 )
874 }
875
876 fn build_pre_tokenized_from_segments(
878 &self,
879 input: &str,
880 segments: &[Segment<'_>],
881 ) -> PreTokenizedString {
882 let normalized_added_tokens = self
883 .added_tokens
884 .as_ref()
885 .filter(|added_tokens| added_tokens.has_normalized());
886
887 if segments.len() == 1
890 && let Segment::Text(text) = segments[0]
891 && normalized_added_tokens.is_none()
892 {
893 let normalized = match &self.normalizer {
894 Some(n) => n.normalize(text),
895 None => std::borrow::Cow::Borrowed(text),
896 };
897 return match normalized {
898 std::borrow::Cow::Borrowed(_) => PreTokenizedString::from_text(text),
899 std::borrow::Cow::Owned(s) => {
900 let len = s.len();
901 PreTokenizedString::new(
902 s,
903 vec![PtSplit {
904 range: 0..len,
905 token_id: None,
906 }],
907 )
908 }
909 };
910 }
911
912 let mut buffer = String::with_capacity(input.len());
913 let mut splits = Vec::new();
914
915 for seg in segments {
916 match seg {
917 Segment::Token(id) => {
918 let start = buffer.len();
919 splits.push(PtSplit {
920 range: start..start,
921 token_id: Some(*id),
922 });
923 }
924 Segment::Text(text) => {
925 if text.is_empty() {
926 continue;
927 }
928 let normalized = match &self.normalizer {
929 Some(n) => n.normalize(text),
930 None => std::borrow::Cow::Borrowed(*text),
931 };
932 if let Some(added_tokens) = normalized_added_tokens {
933 for segment in added_tokens.split_normalized(&normalized) {
936 let start = buffer.len();
937 match segment {
938 Segment::Token(id) => splits.push(PtSplit {
939 range: start..start,
940 token_id: Some(id),
941 }),
942 Segment::Text(text) => {
943 buffer.push_str(text);
944 splits.push(PtSplit {
945 range: start..buffer.len(),
946 token_id: None,
947 });
948 }
949 }
950 }
951 } else {
952 let start = buffer.len();
953 buffer.push_str(&normalized);
954 splits.push(PtSplit {
955 range: start..buffer.len(),
956 token_id: None,
957 });
958 }
959 }
960 }
961 }
962
963 PreTokenizedString::new(buffer, splits)
964 }
965}
966
967fn split_on_unbridgeable_bigrams(
979 pts: &mut PreTokenizedString,
980 bigram_table: &models::bpe::BigramBridgeTable,
981) {
982 let bytes = pts.buffer().as_bytes();
983 let mut new_splits = Vec::with_capacity(pts.splits().len() * 2);
984
985 for split in pts.splits() {
986 if split.token_id.is_some() || split.range.is_empty() {
987 new_splits.push(split.clone());
988 continue;
989 }
990
991 let end = split.range.end;
992 let mut start = split.range.start;
993
994 for i in (start + 1)..end {
995 let prev = bytes[i - 1];
996 let cur = bytes[i];
997
998 if !bigram_table.is_bridgeable(prev, cur) && (cur & 0xC0) != 0x80 {
1002 new_splits.push(PtSplit {
1003 range: start..i,
1004 token_id: None,
1005 });
1006 start = i;
1007 }
1008 }
1009
1010 new_splits.push(PtSplit {
1012 range: start..end,
1013 token_id: None,
1014 });
1015 }
1016
1017 pts.refine_splits(new_splits);
1018}
1019
1020pub struct DecodeStream {
1029 skip_special_tokens: bool,
1030 ids: Vec<u32>,
1031 prefix: String,
1032 prefix_index: usize,
1033}
1034
1035impl DecodeStream {
1036 pub fn new(ids: Vec<u32>, skip_special_tokens: bool) -> Self {
1037 Self {
1038 skip_special_tokens,
1039 ids,
1040 prefix: String::new(),
1041 prefix_index: 0,
1042 }
1043 }
1044
1045 pub fn step(
1046 &mut self,
1047 tokenizer: &Tokenizer,
1048 token_ids: Vec<u32>,
1049 ) -> Result<Option<String>, String> {
1050 decode_stream_step(
1051 tokenizer,
1052 token_ids,
1053 self.skip_special_tokens,
1054 &mut self.ids,
1055 &mut self.prefix,
1056 &mut self.prefix_index,
1057 )
1058 }
1059}
1060
1061pub fn decode_stream_step(
1080 tokenizer: &Tokenizer,
1081 token_ids: Vec<u32>,
1082 skip_special_tokens: bool,
1083 ids: &mut Vec<u32>,
1084 prefix: &mut String,
1085 prefix_index: &mut usize,
1086) -> Result<Option<String>, String> {
1087 const REPLACEMENT: char = '\u{FFFD}';
1088
1089 if prefix.is_empty() && !ids.is_empty() {
1092 let s = tokenizer
1093 .decode(ids, skip_special_tokens)
1094 .map_err(|e| e.to_string())?;
1095 if !s.ends_with(REPLACEMENT) {
1096 *prefix = s;
1097 *prefix_index = ids.len();
1098 }
1099 }
1100
1101 ids.extend(token_ids);
1102
1103 let string = tokenizer
1104 .decode(ids, skip_special_tokens)
1105 .map_err(|e| e.to_string())?;
1106
1107 if string.len() > prefix.len() && !string.ends_with(REPLACEMENT) {
1108 if !string.starts_with(prefix.as_str()) {
1109 return Err(format!(
1110 "Invalid prefix encountered while decoding stream. \
1111 Expected prefix: '{}', Actual string: '{}'",
1112 prefix, string,
1113 ));
1114 }
1115 let new_text = string[prefix.len()..].to_string();
1116 drop(ids.drain(..*prefix_index));
1118 *prefix = tokenizer
1119 .decode(ids, skip_special_tokens)
1120 .map_err(|e| e.to_string())?;
1121 *prefix_index = ids.len();
1122 Ok(Some(new_text))
1123 } else {
1124 Ok(None)
1125 }
1126}
1127
1128#[cfg(test)]
1129mod tests {
1130 use std::path::PathBuf;
1131
1132 use super::*;
1133
1134 const HF_MODELS: &[&str] = &[
1135 "Qwen/Qwen3-0.6B",
1136 "zai-org/GLM-4.7",
1137 "deepseek-ai/DeepSeek-V3.2",
1138 "MiniMaxAI/MiniMax-M2.1",
1139 "openai/gpt-oss-120b",
1140 "mistralai/Mistral-Nemo-Instruct-2407",
1141 "Qwen/Qwen3-235B-A22B-Instruct-2507",
1142 "Qwen/Qwen3-Coder-480B-A35B-Instruct",
1143 "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
1144 "nvidia/Qwen3-Nemotron-235B-A22B-GenRM",
1145 "hoangquan456/Kimi-K2.5",
1146 ];
1147
1148 fn tokenizer_json_path(model: &str) -> anyhow::Result<PathBuf> {
1150 let api = hf_hub::api::sync::Api::new()?;
1151 Ok(api.model(model.to_string()).get("tokenizer.json")?)
1152 }
1153
1154 fn load_tokenizer(model: &str) -> anyhow::Result<Tokenizer> {
1156 let path = tokenizer_json_path(model)?;
1157 Ok(Tokenizer::from_file(&path)?)
1158 }
1159
1160 fn load_reference_tokenizer(model: &str) -> anyhow::Result<tokenizers::Tokenizer> {
1162 let path = tokenizer_json_path(model)?;
1163 tokenizers::Tokenizer::from_file(path).map_err(|error| anyhow::anyhow!(error))
1164 }
1165
1166 #[test]
1168 fn parse_hf_json() {
1169 for model in HF_MODELS {
1170 let json_path = tokenizer_json_path(model).unwrap_or_else(|e| panic!("{model}: {e}"));
1171 let _: TokenizerJson = serde_json::from_str(&fs::read_to_string(json_path).unwrap())
1172 .unwrap_or_else(|e| panic!("{model}: {e}"));
1173 }
1174 }
1175
1176 #[test]
1178 fn encode_batch_matches_sequential() {
1179 let model = "MiniMaxAI/MiniMax-M2.1";
1180 let ours = load_tokenizer(model).unwrap();
1181
1182 let inputs = &["Hello, world!", "The quick brown fox", "Test", ""];
1183 let batch_results = ours.encode_batch(inputs, false).unwrap();
1184
1185 for (input, batch_result) in inputs.iter().zip(&batch_results) {
1186 let sequential_result = ours.encode(input).unwrap();
1187 assert_eq!(
1188 batch_result, &sequential_result,
1189 "batch mismatch for {input:?}"
1190 );
1191 }
1192 }
1193
1194 #[test]
1196 fn encode_batch_ragged_matches_nested() {
1197 let ours = load_tokenizer("openai-community/gpt2").unwrap();
1198 let mut inputs = vec!["Hello, world! ".repeat(32); 64];
1199 inputs[1] = "<|endoftext|>".into();
1200 inputs[2] = String::new();
1201 let expected = ours.encode_batch(&inputs, false).unwrap();
1202 let (actual_ids, actual_lengths) = ours.encode_batch_ragged(&inputs, false).unwrap();
1203
1204 let expected_lengths = expected.iter().map(Vec::len).collect::<Vec<_>>();
1205 let expected_ids = expected.into_iter().flatten().collect::<Vec<_>>();
1206 assert_eq!(actual_lengths, expected_lengths);
1207 assert_eq!(actual_ids, expected_ids);
1208 }
1209
1210 #[test]
1212 fn byte_fallback_merge_crosses_unicode_boundary() {
1213 let tokenizer = Tokenizer::from_json(serde_json::json!({
1214 "normalizer": null,
1215 "pre_tokenizer": null,
1216 "model": {
1217 "type": "BPE",
1218 "vocab": {
1219 "<unk>": 0,
1220 "<0xC3>": 1,
1221 "<0xA9>": 2,
1222 "<0xAA>": 3,
1223 "<0xA9><0xC3>": 4
1224 },
1225 "merges": [["<0xA9>", "<0xC3>"]],
1226 "unk_token": "<unk>",
1227 "byte_fallback": true
1228 },
1229 "post_processor": null,
1230 "decoder": null
1231 }))
1232 .unwrap();
1233
1234 assert_eq!(tokenizer.encode("éê").unwrap(), vec![1, 4, 3]);
1235 }
1236
1237 #[test]
1239 fn normalized_added_tokens_follow_the_two_phase_pipeline() {
1240 let tokenizer = Tokenizer::from_json(serde_json::json!({
1241 "added_tokens": [
1242 {
1243 "id": 10,
1244 "content": "e\u{301}!",
1245 "single_word": false,
1246 "lstrip": false,
1247 "rstrip": false,
1248 "normalized": true,
1249 "special": false
1250 },
1251 {
1252 "id": 11,
1253 "content": "e\u{301}?",
1254 "single_word": false,
1255 "lstrip": false,
1256 "rstrip": false,
1257 "normalized": false,
1258 "special": false
1259 }
1260 ],
1261 "normalizer": { "type": "NFC" },
1262 "pre_tokenizer": null,
1263 "model": {
1264 "type": "BPE",
1265 "vocab": { "<unk>": 0 },
1266 "merges": [],
1267 "unk_token": "<unk>",
1268 "byte_fallback": false
1269 },
1270 "post_processor": null,
1271 "decoder": null
1272 }))
1273 .unwrap();
1274
1275 assert_eq!(tokenizer.encode("é!").unwrap(), vec![10]);
1276 assert_eq!(tokenizer.encode("e\u{301}!").unwrap(), vec![10]);
1277 assert_eq!(tokenizer.encode("e\u{301}?").unwrap(), vec![11]);
1278 }
1279
1280 #[test]
1282 fn vocab_access() {
1283 let model = "MiniMaxAI/MiniMax-M2.1";
1284 let ours = load_tokenizer(model).unwrap();
1285
1286 assert!(ours.vocab_size() > 0);
1287
1288 let token_str = ours.id_to_token(0).expect("token 0 should exist");
1289 let id = ours
1290 .token_to_id(token_str)
1291 .expect("reverse lookup should work");
1292 assert_eq!(id, 0);
1293 }
1294
1295 #[test]
1296 fn public_added_token_accessors_expose_added_vocab() {
1297 let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
1298 let added_tokens = tok.added_tokens().expect("expected added tokens");
1299
1300 let think_id = tok.token_to_id("<think>").expect("<think> should exist");
1301 assert_eq!(added_tokens.token_to_id("<think>"), Some(think_id));
1302 assert_eq!(added_tokens.id_to_token(think_id), Some("<think>"));
1303
1304 let mut entries: Vec<_> = added_tokens.iter().collect();
1305 entries.sort_by_key(|entry| entry.id);
1306 let special_entry = entries
1307 .iter()
1308 .find(|entry| entry.special)
1309 .expect("expected at least one special added token");
1310 assert!(tok.is_special_token(special_entry.id));
1311 assert!(
1312 entries
1313 .iter()
1314 .any(|entry| entry.id == think_id && entry.content == "<think>"),
1315 "added-token iterator should expose <think>"
1316 );
1317 }
1318
1319 const CORPUS: &[&str] = &[
1324 "",
1326 " ",
1327 " ",
1328 "\n",
1329 "\t",
1330 "\r\n",
1331 "a",
1333 "Z",
1334 "0",
1335 "!",
1336 "\u{00e9}", "\u{4e2d}", "Hello, world!",
1340 "The quick brown fox jumps over the lazy dog.",
1341 "A short sentence.",
1342 " leading spaces",
1344 "trailing spaces ",
1345 " both sides ",
1346 "multiple internal spaces",
1347 "tabs\there\tand\tthere",
1348 "line\none\nline\ntwo",
1349 "windows\r\nline\r\nendings",
1350 "mixed\n\ttabs and\r\nnewlines with spaces",
1351 "42",
1353 "3.14159",
1354 "1,000,000",
1355 "0xFF",
1356 "1e-10",
1357 "Numbers 1234567890 and mixed ABC123def",
1358 "Hello!!! How are you???",
1360 "@user #hashtag $100 %50 ^caret & *star",
1361 "a-b_c.d,e;f:g",
1362 "(parentheses) [brackets] {braces}",
1363 "\"double quotes\" 'single quotes' `backticks`",
1364 "path/to/file.txt",
1365 "https://example.com/path?q=test&lang=en#section",
1366 "Special chars: @#$%^&*()_+-=[]{}|;':\",./<>?",
1367 "caf\u{00e9} r\u{00e9}sum\u{00e9} na\u{00ef}ve",
1369 "\u{00fc}ber stra\u{00df}e gr\u{00f6}\u{00df}e",
1370 "se\u{00f1}or ni\u{00f1}o a\u{00f1}o",
1371 "\u{4f60}\u{597d}\u{4e16}\u{754c}", "\u{3053}\u{3093}\u{306b}\u{3061}\u{306f}", "\u{c548}\u{b155}\u{d558}\u{c138}\u{c694}", "\u{041f}\u{0440}\u{0438}\u{0432}\u{0435}\u{0442} \u{043c}\u{0438}\u{0440}",
1377 "\u{0645}\u{0631}\u{062d}\u{0628}\u{0627}",
1379 "\u{0928}\u{092e}\u{0938}\u{094d}\u{0924}\u{0947}",
1381 "\u{1f600}\u{1f680}\u{2764}\u{fe0f}",
1383 "\u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467}\u{200d}\u{1f466}",
1384 "\u{1f1fa}\u{1f1f8}", "e\u{0301}", "n\u{0303}", "a\u{0308}", "Hello \u{4e16}\u{754c} \u{041c}\u{0438}\u{0440}!",
1391 "User123 wrote: \u{4f60}\u{597d}!",
1392 "fn main() { println!(\"hello\"); }",
1394 "def foo(x: int) -> str:\n return str(x)",
1395 "SELECT * FROM users WHERE id = 1;",
1396 "if (x > 0 && y < 10) { z = x + y; }",
1397 "<html><body><p>Hello</p></body></html>",
1398 "#include <stdio.h>\nint main() { return 0; }",
1399 "import numpy as np\nx = np.array([1, 2, 3])",
1400 "{\"key\": \"value\", \"number\": 42, \"array\": [1, 2, 3]}",
1402 "[{\"id\": 1}, {\"id\": 2}]",
1403 "aaaaaaaaaa",
1405 "abababababababab",
1406 "the the the the the the the the",
1407 "....",
1408 "----",
1409 " ",
1410 "\n\n\n\n",
1411 "This is a longer sentence with various elements: numbers (42, 3.14), \
1413 symbols (@#$), Unicode (caf\u{00e9}, \u{4f60}\u{597d}), and more.",
1414 "The year 2024 was notable for advances in AI. Models like GPT-4 and \
1415 Claude demonstrated remarkable capabilities in reasoning, coding, and \
1416 multilingual understanding.",
1417 "a b c d e f g h i j k l m n o p q r s t u v w x y z",
1419 "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
1420 "0123456789",
1421 "a\nb\nc\n",
1423 "# Heading\n\n- item 1\n- item 2\n\n```code```",
1424 "\u{ffff}", "\u{0080}", "\u{07ff}", "\u{0800}", "\u{10000}", "\u{fffd}", "\u{feff}Hello", "\u{0000}", "abc\u{0000}def", "\u{fffe}", "\u{fdd0}", "\u{200b}\u{200c}\u{200d}", "\u{202e}Hello\u{202c}", "\u{0001}\u{0002}\u{001f}\u{007f}", "\u{0300}", "a\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}", "\u{e000}\u{f8ff}", "\u{01c5}\u{01c8}\u{01cb}", "\u{2028}\u{2029}", "\u{fff9}\u{fffa}\u{fffb}", "\u{d7ff}\u{10ffff}", "ab",
1448 "abc",
1449 "abcd",
1450 "aaa",
1451 "aaaa",
1452 "aaaaa",
1453 "**bold** *italic* ~~strikethrough~~ __underline__",
1455 "```rust\nfn main() {}\n```",
1456 "> blockquote\n>> nested",
1457 "| col1 | col2 |\n|------|------|\n| a | b |",
1458 ];
1459
1460 fn compare_encode_decode(model_name: &str, corpus: &[&str]) -> Vec<String> {
1464 let hf = load_reference_tokenizer(model_name)
1465 .unwrap_or_else(|e| panic!("{model_name}: HF load failed: {e}"));
1466 let ours = load_tokenizer(model_name)
1467 .unwrap_or_else(|e| panic!("{model_name}: fastokens load failed: {e}"));
1468
1469 let mut failures = Vec::new();
1470 for &input in corpus {
1471 let hf_enc = hf
1472 .encode(input, false)
1473 .unwrap_or_else(|e| panic!("{model_name}: HF encode({input:?}): {e}"));
1474 let hf_ids = hf_enc.get_ids().to_vec();
1475 let our_ids = match ours.encode(input) {
1476 Ok(ids) => ids,
1477 Err(e) => {
1478 failures.push(format!(" encode error on {input:?}: {e}"));
1479 continue;
1480 }
1481 };
1482 if our_ids != hf_ids {
1483 failures.push(format!(
1484 " encode mismatch on {input:?}: got {} tokens, expected {}\n\
1485 \x20 ours: {:?}\n\
1486 \x20 hf: {:?}",
1487 our_ids.len(),
1488 hf_ids.len(),
1489 &our_ids[..our_ids.len().min(20)],
1490 &hf_ids[..hf_ids.len().min(20)],
1491 ));
1492 }
1493
1494 if input.is_empty() || hf_ids.is_empty() {
1496 continue;
1497 }
1498 let hf_decoded = match hf.decode(&hf_ids, false) {
1499 Ok(d) => d,
1500 Err(_) => continue,
1501 };
1502 let our_decoded = match ours.decode(&hf_ids, false) {
1503 Ok(d) => d,
1504 Err(e) => {
1505 failures.push(format!(" decode error on {input:?}: {e}"));
1506 continue;
1507 }
1508 };
1509 if our_decoded != hf_decoded {
1510 failures.push(format!(
1511 " decode mismatch on {input:?}:\n\
1512 \x20 ours: {:?}\n\
1513 \x20 hf: {:?}",
1514 &our_decoded[..our_decoded.len().min(100)],
1515 &hf_decoded[..hf_decoded.len().min(100)],
1516 ));
1517 }
1518 }
1519 failures
1520 }
1521
1522 #[test]
1525 fn correctness_gpt2() {
1526 let f = compare_encode_decode("openai-community/gpt2", CORPUS);
1527 assert!(f.is_empty(), "openai-community/gpt2:\n{}", f.join("\n"));
1528 }
1529
1530 #[test]
1531 fn correctness_minimax_m2_1() {
1532 let f = compare_encode_decode("MiniMaxAI/MiniMax-M2.1", CORPUS);
1533 assert!(f.is_empty(), "MiniMaxAI/MiniMax-M2.1:\n{}", f.join("\n"));
1534 }
1535
1536 #[test]
1537 fn correctness_nemotron() {
1538 let f = compare_encode_decode("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", CORPUS);
1539 assert!(
1540 f.is_empty(),
1541 "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16:\n{}",
1542 f.join("\n")
1543 );
1544 }
1545
1546 #[test]
1547 fn correctness_deepseek_v3_2() {
1548 let f = compare_encode_decode("deepseek-ai/DeepSeek-V3.2", CORPUS);
1549 assert!(f.is_empty(), "deepseek-ai/DeepSeek-V3.2:\n{}", f.join("\n"));
1550 }
1551
1552 #[test]
1553 fn correctness_gpt_oss() {
1554 let f = compare_encode_decode("openai/gpt-oss-120b", CORPUS);
1555 assert!(f.is_empty(), "openai/gpt-oss-120b:\n{}", f.join("\n"));
1556 }
1557
1558 #[test]
1560 fn ignore_merges_preserves_piece_semantics() {
1561 let tokenizer = |ignore_merges| {
1562 Tokenizer::from_json(serde_json::json!({
1563 "normalizer": null,
1564 "pre_tokenizer": null,
1565 "model": {
1566 "type": "BPE",
1567 "vocab": {"a": 0, "b": 1, "c": 2, "d": 3, "ab": 4, "abc": 5},
1568 "merges": [],
1569 "ignore_merges": ignore_merges
1570 },
1571 "post_processor": null,
1572 "decoder": null
1573 }))
1574 .unwrap()
1575 };
1576
1577 let merged = tokenizer(false);
1578 assert_eq!(merged.encode("ab").unwrap(), vec![0, 1]);
1579
1580 let ignored = tokenizer(true);
1581 assert_eq!(ignored.encode("abc").unwrap(), vec![5]);
1582 assert_eq!(ignored.encode("abd").unwrap(), vec![0, 1, 3]);
1583 }
1584
1585 #[test]
1586 fn ignore_merges_glm47() {
1587 let model = "zai-org/GLM-4.7";
1588 let hf = load_reference_tokenizer(model).unwrap();
1589 let ours = load_tokenizer(model).unwrap();
1590
1591 let text = " имущества";
1595 let hf_ids = hf.encode(text, false).unwrap().get_ids().to_vec();
1596 let our_ids = ours.encode(text).unwrap();
1597 assert_eq!(
1598 our_ids, hf_ids,
1599 "ignore_merges mismatch on {text:?}: ours={our_ids:?} hf={hf_ids:?}"
1600 );
1601
1602 let vocab_size = hf.get_vocab_size(false) as u64;
1604 let random_ids: Vec<u32> = (0..5000)
1605 .map(|i| {
1606 ((i as u64).wrapping_mul(6364136223846793005).wrapping_add(1) % vocab_size) as u32
1607 })
1608 .collect();
1609 let text = hf.decode(&random_ids, true).unwrap();
1610 let hf_enc = hf.encode(text.as_str(), false).unwrap().get_ids().to_vec();
1611 let our_enc = ours.encode(&text).unwrap();
1612 assert_eq!(
1613 our_enc,
1614 hf_enc,
1615 "ignore_merges random-decode mismatch: {} vs {} tokens",
1616 our_enc.len(),
1617 hf_enc.len()
1618 );
1619 }
1620
1621 #[test]
1622 fn correctness_qwen3() {
1623 let f = compare_encode_decode("Qwen/Qwen3-0.6B", CORPUS);
1624 assert!(f.is_empty(), "Qwen/Qwen3-0.6B:\n{}", f.join("\n"));
1625 }
1626
1627 #[test]
1628 fn correctness_mistral_nemo() {
1629 let f = compare_encode_decode("mistralai/Mistral-Nemo-Instruct-2407", CORPUS);
1630 assert!(
1631 f.is_empty(),
1632 "mistralai/Mistral-Nemo-Instruct-2407:\n{}",
1633 f.join("\n")
1634 );
1635 }
1636
1637 #[test]
1638 fn correctness_qwen3_nemotron() {
1639 let f = compare_encode_decode("nvidia/Qwen3-Nemotron-235B-A22B-GenRM", CORPUS);
1640 assert!(
1641 f.is_empty(),
1642 "nvidia/Qwen3-Nemotron-235B-A22B-GenRM:\n{}",
1643 f.join("\n")
1644 );
1645 }
1646
1647 #[test]
1648 fn correctness_kimi_k2_5() {
1649 let f = compare_encode_decode("hoangquan456/Kimi-K2.5", CORPUS);
1650 assert!(f.is_empty(), "hoangquan456/Kimi-K2.5:\n{}", f.join("\n"));
1651 }
1652
1653 #[test]
1658 fn cache_consistency() {
1659 let model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16";
1660 let ours = load_tokenizer(model).unwrap();
1661
1662 let inputs = &[
1663 "Hello, world!",
1664 "The quick brown fox jumps over the lazy dog.",
1665 "caf\u{00e9} r\u{00e9}sum\u{00e9}",
1666 "\u{4f60}\u{597d}\u{4e16}\u{754c}",
1667 "fn main() { println!(\"hello\"); }",
1668 "a b c d e f g h i j k l m n o p",
1669 "aaaaaaaaaa bbbbbbbbbb cccccccccc",
1670 ];
1671
1672 for &input in inputs {
1673 let first = ours.encode(input).unwrap();
1674 let second = ours.encode(input).unwrap();
1675 assert_eq!(first, second, "cache inconsistency for {input:?}");
1676 let third = ours.encode(input).unwrap();
1678 assert_eq!(first, third, "cache inconsistency (3rd call) for {input:?}");
1679 }
1680 }
1681
1682 #[test]
1685 fn cache_consistency_fused() {
1686 let model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16";
1687 let ours = load_tokenizer(model).unwrap();
1688
1689 assert!(
1691 ours.pre_tokenizer
1692 .as_ref()
1693 .and_then(PreTokenizer::fused_byte_level)
1694 .is_some(),
1695 "expected fused path for {model}"
1696 );
1697
1698 let input = "The year 2024 was notable for advances in AI. Models like \
1700 GPT-4 and Claude demonstrated remarkable capabilities.";
1701 let baseline = ours.encode(input).unwrap();
1702 for i in 0..20 {
1703 let result = ours.encode(input).unwrap();
1704 assert_eq!(result, baseline, "fused cache drift on iteration {i}");
1705 }
1706 }
1707
1708 #[test]
1713 fn added_tokens_minimax() {
1714 let corpus = &[
1715 "<filename>",
1716 "open <filename> for reading",
1717 "<filename><reponame>",
1718 "printf(\"%s <filename>\\n\")",
1719 "<think>Let me reason about this.</think>",
1720 "<think>load <filename> from <reponame></think>",
1721 "<file> is not <filename>",
1722 "<fim_prefix>code here<fim_suffix>more code<fim_middle>",
1723 ];
1724 let f = compare_encode_decode("MiniMaxAI/MiniMax-M2.1", corpus);
1725 assert!(
1726 f.is_empty(),
1727 "MiniMaxAI/MiniMax-M2.1 added tokens:\n{}",
1728 f.join("\n")
1729 );
1730 }
1731
1732 #[test]
1734 fn added_tokens_deepseek() {
1735 let corpus = &[
1736 "<|begin▁of▁sentence|>Hello",
1737 "Hello<|end▁of▁sentence|>",
1738 "<|User|>What is 2+2?<|Assistant|>4<|end▁of▁sentence|>",
1739 "Normal text without special tokens",
1740 "<|tool▁calls▁begin|>call<|tool▁calls▁end|>",
1741 ];
1742 let f = compare_encode_decode("deepseek-ai/DeepSeek-V3.2", corpus);
1743 assert!(
1744 f.is_empty(),
1745 "deepseek-ai/DeepSeek-V3.2 added tokens:\n{}",
1746 f.join("\n")
1747 );
1748 }
1749
1750 #[test]
1752 fn added_tokens_qwen3() {
1753 let corpus = &[
1754 "<|im_start|>system\nYou are a helpful assistant.<|im_end|>",
1755 "<|im_start|>user\nHello!<|im_end|>",
1756 "<|endoftext|>",
1757 "Plain text with no special tokens at all.",
1758 ];
1759 let f = compare_encode_decode("Qwen/Qwen3-0.6B", corpus);
1760 assert!(
1761 f.is_empty(),
1762 "Qwen/Qwen3-0.6B added tokens:\n{}",
1763 f.join("\n")
1764 );
1765 }
1766
1767 #[test]
1775 fn token_to_id_searches_added_tokens() {
1776 let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
1777 for token in &[
1779 "<|image_pad|>",
1780 "<|vision_start|>",
1781 "<|vision_end|>",
1782 "<|im_start|>",
1783 ] {
1784 let id = tok.token_to_id(token);
1785 assert!(id.is_some(), "token_to_id({token:?}) returned None");
1786 assert_eq!(tok.id_to_token(id.unwrap()), Some(*token));
1788 }
1789 }
1790
1791 #[test]
1802 fn added_tokens_qwen3vl_vision_sequence() {
1803 let corpus = &[
1804 "<|vision_start|><|image_pad|><|vision_end|>",
1806 "<|image_pad|>",
1808 "<|vision_start|><|image_pad|><|image_pad|><|image_pad|><|image_pad|><|vision_end|>",
1810 "<|vision_start|><|image_pad|><|vision_end|>\nDescribe this image.",
1812 ];
1813 let f = compare_encode_decode("Qwen/Qwen3.5-27B", corpus);
1814 assert!(
1815 f.is_empty(),
1816 "Qwen/Qwen3.5-27B VL vision sequence:\n{}",
1817 f.join("\n")
1818 );
1819 }
1820
1821 #[test]
1823 fn added_tokens_nemotron() {
1824 let corpus = &[
1825 "<|begin_of_text|>Hello world",
1826 "Hello<|end_of_text|>",
1827 "<|start_header_id|>system<|end_header_id|>\n\nYou are helpful.<|eot_id|>",
1828 "<|start_header_id|>user<|end_header_id|>\n\nHi!<|eot_id|>",
1829 "No special tokens here.",
1830 ];
1831 let f = compare_encode_decode("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", corpus);
1832 assert!(
1833 f.is_empty(),
1834 "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 added tokens:\n{}",
1835 f.join("\n")
1836 );
1837 }
1838
1839 #[test]
1844 fn long_input_correctness() {
1845 let model_name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16";
1846 let hf = load_reference_tokenizer(model_name).unwrap();
1847 let ours = load_tokenizer(model_name).unwrap();
1848
1849 let block = "The quick brown fox jumps over the lazy dog. \
1851 Numbers: 42, 3.14, 1000. Code: fn main() {} \
1852 Unicode: caf\u{00e9}, \u{4f60}\u{597d}. \
1853 Special: @#$%^&*(). ";
1854 let input: String = block.repeat(100);
1855 assert!(input.len() > 8000);
1856
1857 let hf_ids = hf.encode(input.as_str(), false).unwrap().get_ids().to_vec();
1858 let our_ids = ours.encode(&input).unwrap();
1859 assert_eq!(
1860 our_ids,
1861 hf_ids,
1862 "long input mismatch: {} vs {} tokens",
1863 our_ids.len(),
1864 hf_ids.len(),
1865 );
1866 }
1867
1868 #[test]
1870 fn newline_partitioned_ragged_matches_sequential() {
1871 let ours = load_tokenizer("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16").unwrap();
1872 assert!(ours.normalizer.is_none());
1873 for input in [
1874 "Text and numbers 123. Unicode: café, 你好. Punctuation?!\r\n\n".repeat(1_200),
1875 "one long line with no partition boundary ".repeat(2_000),
1876 ] {
1877 assert!(input.len() >= PARALLEL_CACHE_BATCH_BYTES);
1878 let expected = ours.encode(&input).unwrap();
1879 let (actual, lengths) = ours.encode_batch_ragged(&[input.as_str()], false).unwrap();
1880 assert_eq!(lengths, [expected.len()]);
1881 assert_eq!(actual, expected);
1882 }
1883 }
1884
1885 #[test]
1887 fn long_input_correctness_minimax() {
1888 let model_name = "MiniMaxAI/MiniMax-M2.1";
1889 let hf = load_reference_tokenizer(model_name).unwrap();
1890 let ours = load_tokenizer(model_name).unwrap();
1891
1892 let block = "The quick brown fox jumps over the lazy dog. \
1893 Numbers: 42, 3.14, 1000. Code: fn main() {} \
1894 Unicode: caf\u{00e9}, \u{4f60}\u{597d}. \
1895 Special: @#$%^&*(). ";
1896 let input: String = block.repeat(100);
1897
1898 let hf_ids = hf.encode(input.as_str(), false).unwrap().get_ids().to_vec();
1899 let our_ids = ours.encode(&input).unwrap();
1900 assert_eq!(
1901 our_ids,
1902 hf_ids,
1903 "long input mismatch: {} vs {} tokens",
1904 our_ids.len(),
1905 hf_ids.len(),
1906 );
1907 }
1908
1909 use std::sync::OnceLock;
1912
1913 struct ExtendedCorpus {
1914 longbench: Vec<String>,
1915 sharegpt: Vec<String>,
1916 }
1917
1918 fn extended_corpus() -> &'static ExtendedCorpus {
1919 static CORPUS: OnceLock<ExtendedCorpus> = OnceLock::new();
1920 CORPUS.get_or_init(|| {
1921 let api = hf_hub::api::sync::Api::new().unwrap();
1922
1923 let lb_repo = api.dataset("zai-org/LongBench-v2".to_string());
1925 let lb_path = lb_repo.get("data.json").unwrap();
1926 let lb_data: Vec<serde_json::Value> =
1927 serde_json::from_str(&fs::read_to_string(lb_path).unwrap()).unwrap();
1928 let longbench: Vec<String> = lb_data
1929 .iter()
1930 .filter_map(|item| {
1931 let ctx = item.get("context")?.as_str()?;
1932 if ctx.is_empty() {
1933 None
1934 } else {
1935 Some(ctx.to_string())
1936 }
1937 })
1938 .take(100)
1939 .collect();
1940
1941 let sg_repo = api.dataset("RyokoAI/ShareGPT52K".to_string());
1943 let sg_path = sg_repo.get("sg_90k_part1.json").unwrap();
1944 let sg_data: Vec<serde_json::Value> =
1945 serde_json::from_str(&fs::read_to_string(sg_path).unwrap()).unwrap();
1946 let sharegpt: Vec<String> = sg_data
1947 .iter()
1948 .filter_map(|item| {
1949 let messages = item.get("conversations")?.as_array()?;
1950 let parts: Vec<String> = messages
1951 .iter()
1952 .filter_map(|msg| {
1953 let role = msg
1954 .get("from")
1955 .and_then(|v| v.as_str())
1956 .unwrap_or("unknown");
1957 let value = msg.get("value").and_then(|v| v.as_str())?;
1958 if value.is_empty() {
1959 return None;
1960 }
1961 Some(format!("[{role}]: {value}"))
1962 })
1963 .collect();
1964 if parts.is_empty() {
1965 None
1966 } else {
1967 Some(parts.join("\n\n"))
1968 }
1969 })
1970 .take(1_000)
1971 .collect();
1972
1973 ExtendedCorpus {
1974 longbench,
1975 sharegpt,
1976 }
1977 })
1978 }
1979
1980 fn compare_encode_decode_batched(
1982 model_name: &str,
1983 corpus: &[String],
1984 batch_size: usize,
1985 progress: bool,
1986 ) -> Vec<String> {
1987 let hf = load_reference_tokenizer(model_name)
1988 .unwrap_or_else(|e| panic!("{model_name}: HF load failed: {e}"));
1989 let ours = load_tokenizer(model_name)
1990 .unwrap_or_else(|e| panic!("{model_name}: fastokens load failed: {e}"));
1991
1992 let total = corpus.len();
1993 let mut processed = 0usize;
1994 let mut failures = Vec::new();
1995 for chunk in corpus.chunks(batch_size) {
1996 let hf_results: Vec<Vec<u32>> = chunk
1997 .iter()
1998 .map(|input| {
1999 hf.encode(input.as_str(), false)
2000 .unwrap_or_else(|e| panic!("{model_name}: HF encode: {e}"))
2001 .get_ids()
2002 .to_vec()
2003 })
2004 .collect();
2005
2006 let our_results = match ours.encode_batch(chunk, false) {
2007 Ok(r) => r,
2008 Err(e) => {
2009 failures.push(format!(" encode_batch error: {e}"));
2010 continue;
2011 }
2012 };
2013
2014 for (i, (hf_ids, our_ids)) in hf_results.iter().zip(our_results.iter()).enumerate() {
2015 let input = &chunk[i];
2016 let input_preview = {
2017 let mut end = input.len().min(80);
2018 while end < input.len() && !input.is_char_boundary(end) {
2019 end += 1;
2020 }
2021 &input[..end]
2022 };
2023
2024 if our_ids != hf_ids {
2025 failures.push(format!(
2026 " encode mismatch on {:?}: got {} tokens, expected {}\n\
2027 \x20 ours: {:?}\n\
2028 \x20 hf: {:?}",
2029 input_preview,
2030 our_ids.len(),
2031 hf_ids.len(),
2032 &our_ids[..our_ids.len().min(20)],
2033 &hf_ids[..hf_ids.len().min(20)],
2034 ));
2035 }
2036
2037 if hf_ids.is_empty() || input.is_empty() {
2039 continue;
2040 }
2041 let hf_decoded = match hf.decode(hf_ids, false) {
2042 Ok(d) => d,
2043 Err(_) => continue,
2044 };
2045 let our_decoded = match ours.decode(hf_ids, false) {
2046 Ok(d) => d,
2047 Err(e) => {
2048 failures.push(format!(" decode error on {input_preview:?}: {e}"));
2049 continue;
2050 }
2051 };
2052 if our_decoded != hf_decoded {
2053 failures.push(format!(
2054 " decode mismatch on {input_preview:?}:\n\
2055 \x20 ours: {:?}\n\
2056 \x20 hf: {:?}",
2057 &our_decoded[..our_decoded.len().min(100)],
2058 &hf_decoded[..hf_decoded.len().min(100)],
2059 ));
2060 }
2061 }
2062 processed += chunk.len();
2063 if progress {
2064 eprint!(
2065 "\r {model_name}: {processed}/{total} ({:.0}%)",
2066 processed as f64 / total as f64 * 100.0,
2067 );
2068 }
2069 }
2070 if progress {
2071 eprintln!();
2072 }
2073 failures
2074 }
2075
2076 fn run_extended(model_name: &str) {
2077 let progress = std::env::var("EXTENDED_PROGRESS").is_ok();
2078 let corpus = extended_corpus();
2079 if progress {
2080 eprintln!(
2081 " {model_name}: longbench ({} samples)",
2082 corpus.longbench.len()
2083 );
2084 }
2085 let mut failures =
2086 compare_encode_decode_batched(model_name, &corpus.longbench, 10, progress);
2087 if progress {
2088 eprintln!(
2089 " {model_name}: sharegpt ({} samples)",
2090 corpus.sharegpt.len()
2091 );
2092 }
2093 failures.extend(compare_encode_decode_batched(
2094 model_name,
2095 &corpus.sharegpt,
2096 10,
2097 progress,
2098 ));
2099 assert!(
2100 failures.is_empty(),
2101 "{model_name} extended ({} failures):\n{}",
2102 failures.len(),
2103 failures.join("\n"),
2104 );
2105 }
2106
2107 #[test]
2108 #[ignore]
2109 fn extended_minimax_m2_1() {
2110 run_extended("MiniMaxAI/MiniMax-M2.1");
2111 }
2112
2113 #[test]
2114 #[ignore]
2115 fn extended_nemotron() {
2116 run_extended("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16");
2117 }
2118
2119 #[test]
2120 #[ignore]
2121 fn extended_deepseek_v3_2() {
2122 run_extended("deepseek-ai/DeepSeek-V3.2");
2123 }
2124
2125 #[test]
2126 #[ignore]
2127 fn extended_gpt_oss() {
2128 run_extended("openai/gpt-oss-120b");
2129 }
2130
2131 #[test]
2132 #[ignore]
2133 fn extended_qwen3() {
2134 run_extended("Qwen/Qwen3-0.6B");
2135 }
2136
2137 #[test]
2138 #[ignore]
2139 fn extended_mistral_nemo() {
2140 run_extended("mistralai/Mistral-Nemo-Instruct-2407");
2141 }
2142
2143 #[test]
2144 #[ignore]
2145 fn extended_qwen3_nemotron() {
2146 run_extended("nvidia/Qwen3-Nemotron-235B-A22B-GenRM");
2147 }
2148
2149 #[test]
2150 #[ignore]
2151 fn extended_mistral_large() {
2152 run_extended("mistralai/Mistral-Large-3-675B-Instruct-2512");
2153 }
2154
2155 #[test]
2156 #[ignore]
2157 fn extended_qwen_small() {
2158 run_extended("Qwen/Qwen3-0.6B");
2159 }
2160
2161 #[test]
2165 fn encode_decode_roundtrip_all_models() {
2166 let texts = &[
2167 "Hello, world!",
2168 "日本語テスト",
2169 "The quick brown fox jumps over the lazy dog.",
2170 "fn main() { println!(\"hello\"); }",
2171 " leading and trailing spaces ",
2172 "line1\nline2\ttabbed",
2173 "0123456789",
2174 "🌍🎉✨",
2175 ];
2176 let failures: Vec<String> = HF_MODELS
2177 .iter()
2178 .flat_map(|model| {
2179 let tok = match load_tokenizer(model) {
2180 Ok(t) => t,
2181 Err(e) => return vec![format!("{model}: load error: {e}")],
2182 };
2183 texts
2184 .iter()
2185 .filter_map(|text| {
2186 let ids = tok.encode_with_special_tokens(text, false).ok()?;
2187 let decoded = tok.decode(&ids, false).ok()?;
2188 if decoded != *text {
2189 Some(format!("{model}: {text:?} → {decoded:?}"))
2190 } else {
2191 None
2192 }
2193 })
2194 .collect()
2195 })
2196 .collect();
2197 assert!(
2198 failures.is_empty(),
2199 "encode→decode roundtrip failures:\n{}",
2200 failures.join("\n")
2201 );
2202 }
2203
2204 #[test]
2215 fn add_bos_token() {
2216 let tok = load_tokenizer("mistralai/Mistral-Nemo-Instruct-2407").unwrap();
2218 let bos_id = tok.token_to_id("<s>").expect("<s> not in vocabulary");
2219
2220 let with_bos = tok.encode_with_special_tokens("hello world", true).unwrap();
2221 let without_bos = tok
2222 .encode_with_special_tokens("hello world", false)
2223 .unwrap();
2224
2225 assert_eq!(
2226 with_bos.first().copied(),
2227 Some(bos_id),
2228 "first token should be BOS when add_special_tokens=true"
2229 );
2230 assert_ne!(
2231 without_bos.first().copied(),
2232 Some(bos_id),
2233 "BOS should be absent when add_special_tokens=false"
2234 );
2235 assert_eq!(&with_bos[1..], without_bos.as_slice());
2237
2238 let tok_q = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2240 let with_flag = tok_q
2241 .encode_with_special_tokens("hello world", true)
2242 .unwrap();
2243 let without_flag = tok_q
2244 .encode_with_special_tokens("hello world", false)
2245 .unwrap();
2246 assert_eq!(
2247 with_flag, without_flag,
2248 "Qwen3 has no BOS post-processor — add_special_tokens should have no effect"
2249 );
2250 }
2251
2252 #[test]
2254 fn decode_skip_special_tokens() {
2255 let model = "mistralai/Mistral-Nemo-Instruct-2407";
2257 let tok = load_tokenizer(model).unwrap();
2258 let text = "hello world";
2259 let ids_with = tok.encode_with_special_tokens(text, true).unwrap();
2260 let ids_without = tok.encode_with_special_tokens(text, false).unwrap();
2261 assert!(
2262 ids_with.len() > ids_without.len(),
2263 "expected BOS/EOS from {model}"
2264 );
2265
2266 let skipped = tok.decode(&ids_with, true).unwrap();
2267 assert_eq!(skipped, text);
2268
2269 let full = tok.decode(&ids_with, false).unwrap();
2270 assert_ne!(full, text);
2271 assert!(full.contains(text));
2272 }
2273
2274 #[test]
2276 fn decode_batch_matches_sequential() {
2277 let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2278 let sentences = &["first sentence", "second sentence", "日本語テスト", ""];
2279 let id_batches: Vec<Vec<u32>> = sentences
2280 .iter()
2281 .map(|s| tok.encode_with_special_tokens(s, false).unwrap())
2282 .collect();
2283 let refs: Vec<&[u32]> = id_batches.iter().map(Vec::as_slice).collect();
2284 let batch_out = tok.decode_batch(&refs, false).unwrap();
2285 for (out, expected) in batch_out.iter().zip(sentences.iter()) {
2286 assert_eq!(out, expected);
2287 }
2288 }
2289
2290 #[test]
2292 fn decode_tokens_matches_decode_by_id() {
2293 let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2294 for text in &["Hello, world!", "The quick brown fox", "🌍 emoji"] {
2295 let ids = tok.encode_with_special_tokens(text, false).unwrap();
2296 let token_strings: Vec<String> = ids
2297 .iter()
2298 .map(|&id| tok.id_to_token(id).unwrap().to_string())
2299 .collect();
2300 let via_ids = tok.decode(&ids, false).unwrap();
2301 let via_tokens = tok.decode_tokens(token_strings).unwrap();
2302 assert_eq!(via_ids, via_tokens, "mismatch for {text:?}");
2303 }
2304 }
2305
2306 #[test]
2308 fn empty_string_encode_decode() {
2309 let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2310 let ids = tok.encode_with_special_tokens("", false).unwrap();
2311 assert!(ids.is_empty(), "expected no tokens for empty string");
2312 assert_eq!(tok.decode(&[], false).unwrap(), "");
2313 }
2314
2315 #[test]
2317 fn encode_is_stable_after_decode() {
2318 let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2319 for text in &["hello world", "日本語テスト", "fn foo() {}"] {
2320 let ids1 = tok.encode_with_special_tokens(text, false).unwrap();
2321 let decoded = tok.decode(&ids1, false).unwrap();
2322 let ids2 = tok.encode_with_special_tokens(&decoded, false).unwrap();
2323 assert_eq!(ids1, ids2, "encode not stable after decode for {text:?}");
2324 }
2325 }
2326
2327 #[test]
2329 fn post_process_false_is_identity_all_models() {
2330 for model in HF_MODELS {
2331 let tok = load_tokenizer(model).unwrap();
2332 let payload = vec![100u32, 200, 300];
2333 let out = tok.post_process(payload.clone(), false);
2334 assert_eq!(
2335 out, payload,
2336 "{model}: post_process(false) should be identity"
2337 );
2338 }
2339 }
2340
2341 #[test]
2343 fn post_process_true_adds_special_tokens() {
2344 let tok = load_tokenizer("mistralai/Mistral-Nemo-Instruct-2407").unwrap();
2346 let payload = vec![10u32, 20, 30];
2347 let without = tok.post_process(payload.clone(), false);
2348 let with_sp = tok.post_process(payload.clone(), true);
2349 assert_eq!(without, payload);
2350 assert!(
2351 with_sp.len() > without.len(),
2352 "expected special tokens to be added"
2353 );
2354 assert!(
2356 with_sp
2357 .windows(payload.len())
2358 .any(|w| w == payload.as_slice()),
2359 "payload should appear contiguously in post-processed output"
2360 );
2361 }
2362
2363 #[test]
2365 fn decode_unknown_id_is_skipped() {
2366 let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2367 assert_eq!(tok.decode(&[u32::MAX], false).unwrap(), "");
2368 }
2369
2370 #[test]
2372 fn decode_mixed_valid_and_unknown_ids() {
2373 let tok = load_tokenizer("Qwen/Qwen3-0.6B").unwrap();
2374 let valid = tok.encode_with_special_tokens("hello", false).unwrap();
2375 let mut mixed = valid.clone();
2376 mixed.push(u32::MAX);
2377 mixed.extend(tok.encode_with_special_tokens(" world", false).unwrap());
2378 let expected = tok.decode(&valid, false).unwrap()
2379 + &tok
2380 .decode(
2381 &tok.encode_with_special_tokens(" world", false).unwrap(),
2382 false,
2383 )
2384 .unwrap();
2385 assert_eq!(tok.decode(&mixed, false).unwrap(), expected);
2386 }
2387
2388 #[test]
2390 fn token_id_roundtrip_all_models() {
2391 let probe_ids = [0u32, 1, 2, 100, 1000, 10_000];
2392 let failures: Vec<String> = HF_MODELS
2393 .iter()
2394 .flat_map(|model| {
2395 let tok = match load_tokenizer(model) {
2396 Ok(t) => t,
2397 Err(e) => return vec![format!("{model}: load error: {e}")],
2398 };
2399 probe_ids
2400 .iter()
2401 .filter_map(|&id| {
2402 let token = tok.id_to_token(id)?;
2403 let back = tok.token_to_id(token)?;
2404 if back != id {
2405 Some(format!("{model}: id {id} → {token:?} → {back}"))
2406 } else {
2407 None
2408 }
2409 })
2410 .collect()
2411 })
2412 .collect();
2413 assert!(
2414 failures.is_empty(),
2415 "id↔token roundtrip failures:\n{}",
2416 failures.join("\n")
2417 );
2418 }
2419
2420 const STREAM_MODEL: &str = "Qwen/Qwen3-0.6B";
2423
2424 fn stream_tok() -> Tokenizer {
2425 load_tokenizer(STREAM_MODEL).expect("failed to load tokenizer")
2426 }
2427
2428 fn stream_collect(tok: &Tokenizer, ids: &[u32], skip: bool) -> (String, usize) {
2429 let mut buf = Vec::new();
2430 let mut prefix = String::new();
2431 let mut prefix_index = 0usize;
2432 let mut out = String::new();
2433 for &id in ids {
2434 let chunk: Option<String> = super::decode_stream_step(
2435 tok,
2436 vec![id],
2437 skip,
2438 &mut buf,
2439 &mut prefix,
2440 &mut prefix_index,
2441 )
2442 .unwrap();
2443 if let Some(c) = chunk {
2444 out.push_str(&c);
2445 }
2446 }
2447 (out, buf.len())
2448 }
2449
2450 #[test]
2451 fn decode_stream_reconstructs_ascii() {
2452 let tok = stream_tok();
2453 let text = "Hello, world! This is a streaming decode test.";
2454 let ids = tok.encode_with_special_tokens(text, false).unwrap();
2455 let (decoded, _) = stream_collect(&tok, &ids, false);
2456 assert_eq!(decoded, text);
2457 }
2458
2459 #[test]
2460 fn decode_stream_reconstructs_unicode() {
2461 let tok = stream_tok();
2462 let text = "日本語テスト: こんにちは 🌍 — привет мир";
2463 let ids = tok.encode_with_special_tokens(text, false).unwrap();
2464 let (decoded, _) = stream_collect(&tok, &ids, false);
2465 assert_eq!(decoded, text);
2466 }
2467
2468 #[test]
2469 fn decode_stream_reconstructs_code() {
2470 let tok = stream_tok();
2471 let text = r#"fn main() { println!("hello"); }"#;
2472 let ids = tok.encode_with_special_tokens(text, false).unwrap();
2473 let (decoded, _) = stream_collect(&tok, &ids, false);
2474 assert_eq!(decoded, text);
2475 }
2476
2477 #[test]
2478 fn decode_stream_empty_ids_no_output() {
2479 let tok = stream_tok();
2480 let (decoded, buf_len) = stream_collect(&tok, &[], false);
2481 assert!(decoded.is_empty());
2482 assert_eq!(buf_len, 0);
2483 }
2484
2485 #[test]
2486 fn decode_stream_single_token() {
2487 let tok = stream_tok();
2488 let ids = tok.encode_with_special_tokens("hello", false).unwrap();
2489 assert!(!ids.is_empty());
2490 let (decoded, _) = stream_collect(&tok, &ids[..1], false);
2491 assert!(!decoded.is_empty());
2492 }
2493
2494 #[test]
2495 fn decode_stream_batch_step_matches_sequential() {
2496 let tok = stream_tok();
2497 let text = "The quick brown fox jumps over the lazy dog.";
2498 let ids = tok.encode_with_special_tokens(text, false).unwrap();
2499 let (sequential, _) = stream_collect(&tok, &ids, false);
2500 let mut buf = Vec::new();
2501 let mut prefix = String::new();
2502 let mut prefix_index = 0usize;
2503 let batch: String = super::decode_stream_step(
2504 &tok,
2505 ids.clone(),
2506 false,
2507 &mut buf,
2508 &mut prefix,
2509 &mut prefix_index,
2510 )
2511 .unwrap()
2512 .unwrap_or_default();
2513 assert_eq!(sequential, batch);
2514 }
2515
2516 #[test]
2517 fn decode_stream_pre_seeded_only_returns_new_tokens() {
2518 let tok = stream_tok();
2519 let prompt = "The capital of France is";
2520 let cont = " Paris.";
2521 let prompt_ids = tok.encode_with_special_tokens(prompt, false).unwrap();
2522 let cont_ids = tok.encode_with_special_tokens(cont, false).unwrap();
2523 let mut buf = prompt_ids.clone();
2524 let mut prefix = String::new();
2525 let mut prefix_index = 0usize;
2526 let mut out = String::new();
2527 for &id in &cont_ids {
2528 let chunk: Option<String> = super::decode_stream_step(
2529 &tok,
2530 vec![id],
2531 false,
2532 &mut buf,
2533 &mut prefix,
2534 &mut prefix_index,
2535 )
2536 .unwrap();
2537 if let Some(c) = chunk {
2538 out.push_str(&c);
2539 }
2540 }
2541 assert_eq!(out, cont);
2542 }
2543
2544 #[test]
2545 fn decode_stream_skip_special_tokens() {
2546 let tok = load_tokenizer("mistralai/Mistral-Nemo-Instruct-2407").unwrap();
2547 let text = "hello";
2548 let ids_with = tok.encode_with_special_tokens(text, true).unwrap();
2549 let ids_without = tok.encode_with_special_tokens(text, false).unwrap();
2550 assert!(
2551 ids_with.len() > ids_without.len(),
2552 "expected BOS/EOS tokens"
2553 );
2554 let (with_sp, _) = stream_collect(&tok, &ids_with, false);
2555 let (no_sp, _) = stream_collect(&tok, &ids_with, true);
2556 assert_eq!(no_sp, text);
2557 assert!(with_sp.contains(&no_sp));
2558 }
2559
2560 #[test]
2561 fn decode_stream_buffer_does_not_grow_unboundedly() {
2562 let tok = stream_tok();
2563 let text = "word ".repeat(80);
2564 let ids = tok.encode_with_special_tokens(text.trim(), false).unwrap();
2565 let (_, final_buf_len) = stream_collect(&tok, &ids, false);
2566 assert!(
2567 final_buf_len < 10,
2568 "buffer grew to {final_buf_len} entries after {} tokens",
2569 ids.len()
2570 );
2571 }
2572
2573 #[test]
2574 fn decode_stream_chunks_are_non_empty_and_concatenate() {
2575 let tok = stream_tok();
2576 let text = "one two three four five six seven eight nine ten";
2577 let ids = tok.encode_with_special_tokens(text, false).unwrap();
2578 let mut buf = Vec::new();
2579 let mut prefix = String::new();
2580 let mut prefix_index = 0usize;
2581 let mut chunks: Vec<String> = Vec::new();
2582 for &id in &ids {
2583 let chunk: Option<String> = super::decode_stream_step(
2584 &tok,
2585 vec![id],
2586 false,
2587 &mut buf,
2588 &mut prefix,
2589 &mut prefix_index,
2590 )
2591 .unwrap();
2592 if let Some(c) = chunk {
2593 assert!(!c.is_empty(), "stream emitted an empty chunk");
2594 chunks.push(c);
2595 }
2596 }
2597 assert_eq!(chunks.concat(), text);
2598 }
2599
2600 #[test]
2605 fn decode_stream_unknown_id_does_not_error() {
2606 let tok = stream_tok();
2607 let mut buf = Vec::new();
2608 let mut prefix = String::new();
2609 let mut prefix_index = 0usize;
2610 let result = super::decode_stream_step(
2611 &tok,
2612 vec![u32::MAX],
2613 false,
2614 &mut buf,
2615 &mut prefix,
2616 &mut prefix_index,
2617 );
2618 assert!(result.is_ok(), "expected Ok, got {result:?}");
2619 }
2620
2621 #[test]
2622 fn decode_stream_invalid_prefix_error_message() {
2623 let tok = stream_tok();
2624 let ids = tok.encode_with_special_tokens("hello", false).unwrap();
2625 let mut buf = ids.clone();
2626 let mut prefix = "ZZZZZZZ".to_string();
2627 let mut prefix_index = 0usize;
2628 let result: Result<Option<String>, String> = super::decode_stream_step(
2629 &tok,
2630 vec![*ids.last().unwrap()],
2631 false,
2632 &mut buf,
2633 &mut prefix,
2634 &mut prefix_index,
2635 );
2636 if let Err(msg) = result {
2637 assert!(
2638 msg.starts_with("Invalid prefix encountered"),
2639 "unexpected error: {msg:?}"
2640 );
2641 }
2642 }
2643}