1pub mod basetenkenizer;
5pub mod cache;
6pub mod fastokens;
7pub mod hf;
8pub mod tekken;
9pub mod tiktoken;
10
11use std::hash::{DefaultHasher, Hash, Hasher};
16use std::sync::Arc;
17use std::{fs::File, io::BufReader, ops::Deref, path::Path};
18
19use anyhow::Context as _;
20pub use anyhow::{Error, Result};
21
22pub use basetenkenizer::BasetenTokenizer;
23pub use cache::{
24 CacheTokenUsage, CacheTokenUsageFn, CachedTokenizer, L1CacheStats, SharedTokenizerCache,
25 SharedTokenizerCacheStats,
26};
27pub use fastokens::{FastTikTokenTokenizer, FastTokenizer};
28pub use hf::HuggingFaceTokenizer;
29pub use tekken::TekkenTokenizer;
30pub use tiktoken::TikTokenTokenizer;
31pub use traits::DecodeResult;
32
33pub type TokenIdType = u32;
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
37pub struct EncodeSegment<'a> {
38 pub text: &'a str,
39 pub allow_special: bool,
44}
45
46impl<'a> EncodeSegment<'a> {
47 pub const fn new(text: &'a str, allow_special: bool) -> Self {
48 Self {
49 text,
50 allow_special,
51 }
52 }
53
54 pub const fn ordinary(text: &'a str) -> Self {
55 Self::new(text, false)
56 }
57
58 pub const fn control(text: &'a str) -> Self {
59 Self::new(text, true)
60 }
61}
62
63#[derive(Debug)]
65pub enum TokenizerType {
66 HuggingFace(String),
67 TikToken(String),
68}
69
70pub type Offsets = (usize, usize);
72
73#[derive(Debug, Clone)]
75pub enum Encoding {
76 Hf(Box<tokenizers::tokenizer::Encoding>),
78 Sp(Vec<TokenIdType>),
80}
81
82impl Encoding {
83 pub fn token_ids(&self) -> &[u32] {
84 match self {
85 Encoding::Hf(inner) => inner.get_ids(),
86 Encoding::Sp(inner) => inner,
87 }
88 }
89}
90
91impl Hash for Encoding {
92 fn hash<H: Hasher>(&self, state: &mut H) {
93 self.token_ids().hash(state);
94 }
95}
96
97pub mod traits {
98 use super::*;
99
100 pub trait Encoder: Send + Sync {
101 fn encode(&self, input: &str) -> Result<Encoding>;
102 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>>;
103
104 fn encode_segments(&self, _segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
125 Err(Error::msg(
126 "tokenizer backend does not support segmented encoding",
127 ))
128 }
129 }
130
131 #[derive(Debug, Clone, PartialEq, Eq, strum::EnumIs)]
138 pub enum DecodeResult {
139 Complete(String),
143 Partial(String),
146 }
147
148 impl DecodeResult {
149 pub fn as_str(&self) -> &str {
151 match self {
152 DecodeResult::Complete(s) | DecodeResult::Partial(s) => s,
153 }
154 }
155
156 pub fn from_decoded(text: String) -> Self {
158 if text.ends_with('\u{FFFD}') {
159 DecodeResult::Partial(text)
160 } else {
161 DecodeResult::Complete(text)
162 }
163 }
164 }
165
166 impl From<String> for DecodeResult {
167 fn from(text: String) -> Self {
168 DecodeResult::from_decoded(text)
169 }
170 }
171
172 impl From<DecodeResult> for String {
173 fn from(result: DecodeResult) -> Self {
174 match result {
175 DecodeResult::Complete(s) | DecodeResult::Partial(s) => s,
176 }
177 }
178 }
179
180 pub trait Decoder: Send + Sync {
186 fn has_unstable_suffix(
189 &self,
190 _token_ids: &[TokenIdType],
191 _skip_special_tokens: bool,
192 ) -> bool {
193 false
194 }
195
196 fn decode(
197 &self,
198 token_ids: &[TokenIdType],
199 skip_special_tokens: bool,
200 ) -> Result<DecodeResult>;
201 }
202
203 pub trait Tokenizer: Encoder + Decoder {
204 fn validate_prefix_cache(&self) -> Result<()> {
209 Err(Error::msg("tokenizer does not support prefix caching"))
210 }
211
212 fn with_options(self, options: TokenizerOptions) -> Self
217 where
218 Self: Sized,
219 {
220 let _ = options;
221 self
222 }
223 fn vocab_size(&self) -> Option<usize> {
227 None
228 }
229
230 fn token_to_id(&self, _token: &str) -> Result<Option<TokenIdType>> {
240 Err(Error::msg(
241 "tokenizer backend does not support token lookup",
242 ))
243 }
244
245 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
255 Err(Error::msg(
256 "tokenizer backend does not support special token enumeration",
257 ))
258 }
259
260 fn num_special_tokens_added(&self) -> Result<usize> {
271 Err(Error::msg(
272 "tokenizer backend does not support special token accounting",
273 ))
274 }
275 }
277}
278
279pub fn file_json_field<T: serde::de::DeserializeOwned>(
280 json_file_path: &Path,
281 field_name: &str,
282) -> anyhow::Result<T> {
283 let file = File::open(json_file_path)
284 .with_context(|| format!("Failed to open file: {:?}", json_file_path))?;
285 let reader = BufReader::new(file);
286
287 let json_data: serde_json::Value = serde_json::from_reader(reader)
288 .with_context(|| format!("Failed to parse JSON from file: {:?}", json_file_path))?;
289
290 let map = json_data.as_object().ok_or_else(|| {
291 anyhow::anyhow!("JSON root is not an object in file: {:?}", json_file_path)
292 })?;
293
294 let field_value = map.get(field_name).ok_or_else(|| {
295 anyhow::anyhow!(
296 "Field '{}' not found in JSON file: {:?}",
297 field_name,
298 json_file_path
299 )
300 })?;
301
302 serde_json::from_value(field_value.clone()).with_context(|| {
303 format!(
304 "Failed to deserialize field '{}' (value: {:?}) to the expected type from file: {:?}",
305 field_name, field_value, json_file_path
306 )
307 })
308}
309
310pub fn log_json_err(filename: &str, json: &str, err: &serde_json::Error) {
311 const ERROR_PREFIX: &str = ">> ";
312
313 if !(err.is_syntax() || err.is_data()) {
314 return;
315 }
316
317 let line = err.line().saturating_sub(1);
318 let column = err.column().saturating_sub(1);
319
320 let json_lines: Vec<&str> = json.lines().collect();
321 if json_lines.is_empty() {
322 tracing::error!("JSON parsing error in {filename}: File is empty.");
323 return;
324 }
325
326 let start_index = line.saturating_sub(2);
327 let end_index = line.saturating_add(3).min(json_lines.len());
328
329 let mut context_lines: Vec<String> = (start_index..end_index)
330 .map(|i| {
331 if i == line {
332 format!("{ERROR_PREFIX}{}", json_lines[i])
333 } else {
334 format!("{:06} {}", i + 1, json_lines[i])
335 }
336 })
337 .collect();
338
339 let col_indicator = "_".to_string().repeat(column + ERROR_PREFIX.len()) + "^";
340 let error_in_context_idx = line - start_index;
341 if error_in_context_idx < context_lines.len() {
342 context_lines.insert(error_in_context_idx + 1, col_indicator);
343 }
344
345 tracing::error!(
346 "JSON parsing error in {filename}: Line {}, column {}:\n{}",
347 err.line(),
348 err.column(),
349 context_lines.join("\n")
350 );
351}
352
353impl Encoding {
354 pub fn get_hash(&self) -> u64 {
355 let mut hasher = DefaultHasher::new();
356 self.hash(&mut hasher);
357 hasher.finish()
358 }
359}
360
361#[derive(Debug, Clone, Copy, Default)]
365pub struct TokenizerOptions {
366 pub add_special_tokens: bool,
373}
374
375#[derive(Clone)]
377pub struct Tokenizer(Arc<dyn traits::Tokenizer>);
378
379impl Tokenizer {
380 pub fn from_file(file_path: &str) -> Result<Tokenizer> {
381 Ok(Tokenizer(create_tokenizer_from_file(file_path)?))
382 }
383
384 pub fn from_file_with_options(file_path: &str, options: TokenizerOptions) -> Result<Tokenizer> {
385 Ok(Tokenizer(create_tokenizer_from_file_with_options(
386 file_path, options,
387 )?))
388 }
389
390 pub fn decode_stream(
394 &self,
395 prompt_token_ids: &[TokenIdType],
396 skip_special_tokens: bool,
397 ) -> DecodeStream {
398 DecodeStream::new(self.0.clone(), prompt_token_ids, skip_special_tokens)
399 }
400}
401
402impl Deref for Tokenizer {
403 type Target = Arc<dyn traits::Tokenizer>;
404
405 fn deref(&self) -> &Self::Target {
406 &self.0
407 }
408}
409
410impl From<Arc<dyn traits::Tokenizer>> for Tokenizer {
411 fn from(tokenizer: Arc<dyn traits::Tokenizer>) -> Self {
412 Tokenizer(tokenizer)
413 }
414}
415
416impl<T> From<Arc<T>> for Tokenizer
417where
418 T: traits::Tokenizer + 'static, {
420 fn from(tokenizer: Arc<T>) -> Self {
421 Tokenizer(tokenizer)
422 }
423}
424
425pub fn create_tokenizer_from_file(file_path: &str) -> Result<Arc<dyn traits::Tokenizer>> {
433 create_tokenizer_from_file_with_options(file_path, Default::default())
434}
435
436pub fn create_tokenizer_from_file_with_options(
444 file_path: &str,
445 options: TokenizerOptions,
446) -> Result<Arc<dyn traits::Tokenizer>> {
447 use traits::Tokenizer as _;
448
449 let path = Path::new(file_path);
450 if TekkenTokenizer::is_tekken_file(path) {
451 let tokenizer = TekkenTokenizer::from_file(file_path)?.with_options(options);
452 return Ok(Arc::new(tokenizer));
453 }
454
455 let extension = path
456 .extension()
457 .and_then(std::ffi::OsStr::to_str)
458 .ok_or_else(|| Error::msg("Failed to read file extension".to_string()))?;
459
460 match extension {
461 "json" => {
462 let tokenizer = HuggingFaceTokenizer::from_file(file_path)?.with_options(options);
463 Ok(Arc::new(tokenizer))
464 }
465 "model" | "tiktoken" => {
466 let tokenizer = TikTokenTokenizer::from_file_auto(file_path)?.with_options(options);
467 Ok(Arc::new(tokenizer))
468 }
469 _ => Err(Error::msg(format!(
470 "Unsupported tokenizer file type: .{extension}"
471 ))),
472 }
473}
474
475const INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET: usize = 5;
481
482pub struct DecodeStream {
488 tokenizer: Arc<dyn traits::Tokenizer>,
490
491 skip_special_tokens: bool,
492 all_token_ids: Vec<u32>,
504
505 prefix_offset: usize,
506
507 read_offset: usize,
508
509 has_emitted: bool,
511}
512
513impl DecodeStream {
514 pub fn new(
515 tokenizer: Arc<dyn traits::Tokenizer>,
516 prompt_token_ids: &[TokenIdType],
517 skip_special_tokens: bool,
518 ) -> Self {
519 let context_start = prompt_token_ids
522 .len()
523 .saturating_sub(INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET);
524 let prompt_token_ids = prompt_token_ids[context_start..].to_vec();
525 let num_input_tokens = prompt_token_ids.len();
526 Self {
527 tokenizer,
528 skip_special_tokens,
529 all_token_ids: prompt_token_ids,
530 prefix_offset: 0,
531 read_offset: num_input_tokens,
532 has_emitted: false,
533 }
534 }
535
536 pub fn step(&mut self, id: u32) -> Result<Option<String>> {
549 self.all_token_ids.push(id);
550
551 if self.tokenizer.has_unstable_suffix(
552 &self.all_token_ids[self.read_offset..],
553 self.skip_special_tokens,
554 ) {
555 return Ok(None);
556 }
557 self.decode_pending(false)
558 }
559
560 pub fn finish(&mut self) -> Result<Option<String>> {
564 self.decode_pending(true)
565 }
566
567 fn decode_pending(&mut self, finishing: bool) -> Result<Option<String>> {
568 if self.read_offset == self.all_token_ids.len() {
569 return Ok(None);
570 }
571
572 let prefix_text: String = self
573 .tokenizer
574 .decode(
575 &self.all_token_ids[self.prefix_offset..self.read_offset],
576 self.skip_special_tokens,
577 )?
578 .into();
579
580 let new_result = self.tokenizer.decode(
581 &self.all_token_ids[self.prefix_offset..],
582 self.skip_special_tokens,
583 )?;
584
585 let new_text = new_result.as_str();
586 let is_partial = new_result.is_partial() && !finishing;
587
588 if self.has_emitted && !is_partial && !new_text.starts_with(prefix_text.as_str()) {
591 return Err(Error::msg(
592 "incremental decoding rewrote already emitted text",
593 ));
594 }
595
596 if new_text.len() > prefix_text.len() && !is_partial {
597 let requested_split = prefix_text.len();
598 let split = if self.has_emitted {
599 requested_split
602 } else {
603 new_text.floor_char_boundary(requested_split)
606 };
607
608 let emitted = new_text[split..].to_string();
609
610 self.prefix_offset = self.read_offset;
611 self.read_offset = self.all_token_ids.len();
612 self.has_emitted = true;
613
614 Ok(Some(emitted))
615 } else {
616 Ok(None)
617 }
618 }
619}
620
621#[cfg(test)]
622mod decode_stream_unicode_tests {
623 use super::{DecodeResult, DecodeStream, Encoding, Result, TokenIdType};
624 use std::sync::Arc;
625
626 struct RewritingTokenizer {
627 prefix_text: &'static str,
628 rewritten_text: &'static str,
629 prefix_is_partial: bool,
630 }
631
632 impl super::traits::Encoder for RewritingTokenizer {
633 fn encode(&self, _input: &str) -> Result<Encoding> {
634 Ok(Encoding::Sp(vec![]))
635 }
636
637 fn encode_batch(&self, _inputs: &[&str]) -> Result<Vec<Encoding>> {
638 Ok(vec![])
639 }
640 }
641
642 impl super::traits::Decoder for RewritingTokenizer {
643 fn decode(
644 &self,
645 token_ids: &[TokenIdType],
646 _skip_special_tokens: bool,
647 ) -> Result<DecodeResult> {
648 let result = match token_ids.len() {
649 1 if self.prefix_is_partial => DecodeResult::Partial(self.prefix_text.to_string()),
650 1 => DecodeResult::Complete(self.prefix_text.to_string()),
651 2 => DecodeResult::Complete(self.rewritten_text.to_string()),
652 _ => DecodeResult::Complete(String::new()),
653 };
654 Ok(result)
655 }
656 }
657
658 impl super::traits::Tokenizer for RewritingTokenizer {}
659
660 #[test]
661 fn prompt_suffix_preserves_decode_inputs_and_output() {
662 let tokenizer: Arc<dyn super::traits::Tokenizer> = Arc::new(
663 super::HuggingFaceTokenizer::from_file(concat!(
664 env!("CARGO_MANIFEST_DIR"),
665 "/tests/data/minimal-bpe/tokenizer.json"
666 ))
667 .unwrap(),
668 );
669 for prompt_len in [0usize, 1, 4, 5, 6, 775_168] {
670 let prompt: Vec<_> = (0..prompt_len)
671 .map(|index| 1 + (index % 22) as u32)
672 .collect();
673 for skip_special_tokens in [false, true] {
674 let mut stream = DecodeStream::new(tokenizer.clone(), &prompt, skip_special_tokens);
675 let mut full = DecodeStream {
677 tokenizer: tokenizer.clone(),
678 skip_special_tokens,
679 all_token_ids: prompt.clone(),
680 prefix_offset: prompt_len
681 .saturating_sub(super::INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET),
682 read_offset: prompt_len,
683 has_emitted: false,
684 };
685 assert!(stream.all_token_ids.capacity() <= 5);
686 for token in [5, 9, 12, 12, 13, 0, 1, 17, 13, 14, 12, 8, 2] {
687 assert_eq!(
688 &stream.all_token_ids[stream.prefix_offset..stream.read_offset],
689 &full.all_token_ids[full.prefix_offset..full.read_offset],
690 );
691 assert_eq!(
692 &stream.all_token_ids[stream.prefix_offset..],
693 &full.all_token_ids[full.prefix_offset..],
694 );
695 let actual = stream.step(token).map_err(|error| error.to_string());
696 let expected = full.step(token).map_err(|error| error.to_string());
697 assert_eq!(actual, expected, "prompt length {prompt_len}");
698 if actual.is_err() {
699 break;
700 }
701 }
702 }
703 }
704 }
705
706 #[test]
707 fn allows_boundary_recovery_before_generated_text_is_emitted() {
708 for (prefix_text, rewritten_text, expected) in [
709 ("㺄馉凓鄗\u{FFFD}", "㺄馉凓鄗𫷲", "𫷲"),
710 (
711 "JUnitworkflow Completion intuition\u{FFFD}",
712 "JUnitworkflow Completion intuition𝟙",
713 "𝟙",
714 ),
715 ] {
716 let tokenizer: Arc<dyn super::traits::Tokenizer> = Arc::new(RewritingTokenizer {
717 prefix_text,
718 rewritten_text,
719 prefix_is_partial: true,
720 });
721 let mut stream = DecodeStream::new(tokenizer, &[1], false);
722
723 assert_eq!(stream.step(2).unwrap(), Some(expected.to_string()));
724 }
725 }
726
727 #[test]
728 fn errors_when_incremental_decode_rewrites_emitted_text() {
729 for (prefix_text, rewritten_text) in [
730 ("abcde", "abcdXY"),
731 ("abcde", "abcdX"),
732 ("abcde", "abcd"),
733 ("㺄馉凓鄗abc", "㺄馉凓鄗𫷲"),
734 (
735 "JUnitworkflow Completion intuitionabc",
736 "JUnitworkflow Completion intuition𝟙",
737 ),
738 ] {
739 let tokenizer: Arc<dyn super::traits::Tokenizer> = Arc::new(RewritingTokenizer {
740 prefix_text,
741 rewritten_text,
742 prefix_is_partial: false,
743 });
744 let mut stream = DecodeStream::new(tokenizer, &[], false);
745
746 assert_eq!(stream.step(1).unwrap(), Some(prefix_text.to_string()));
747
748 let error = stream.step(2).unwrap_err();
749 assert!(error.to_string().contains("already emitted text"));
750 }
751 }
752
753 #[test]
754 fn emits_suffix_when_incremental_decode_preserves_emitted_text() {
755 let tokenizer: Arc<dyn super::traits::Tokenizer> = Arc::new(RewritingTokenizer {
756 prefix_text: "abcde",
757 rewritten_text: "abcdeXY",
758 prefix_is_partial: false,
759 });
760 let mut stream = DecodeStream::new(tokenizer, &[], false);
761
762 assert_eq!(stream.step(1).unwrap(), Some("abcde".to_string()));
763 assert_eq!(stream.step(2).unwrap(), Some("XY".to_string()));
764 }
765}
766
767pub struct Sequence {
769 tokenizer: Tokenizer,
771
772 token_ids: Vec<TokenIdType>,
774
775 prefix_offset: usize,
777
778 read_offset: usize,
780}
781
782impl std::fmt::Debug for Sequence {
783 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
784 f.debug_struct("Sequence")
785 .field("tokenizer", &"Arc<dyn Tokenizer>")
786 .field(
787 "token_ids",
788 &format_args!("{}", {
789 let token_ids = self.token_ids();
790 if token_ids.len() <= 20 {
791 format!("{:?}", token_ids)
792 } else {
793 let first_ten = &token_ids[..10];
794 let last_ten = &token_ids[token_ids.len() - 10..];
795 format!("{:?} ... {:?}", first_ten, last_ten)
796 }
797 }),
798 )
799 .field("prefix_offset", &self.prefix_offset)
800 .field("read_offset", &self.read_offset)
801 .field("token count", &self.token_ids.len())
802 .finish()
803 }
804}
805
806impl Sequence {
807 pub fn new(tokenizer: Tokenizer) -> Self {
808 Self {
809 tokenizer,
810 token_ids: Vec::new(),
811 prefix_offset: 0,
812 read_offset: 0,
813 }
814 }
815
816 pub fn is_empty(&self) -> bool {
817 self.token_ids.is_empty()
818 }
819
820 pub fn len(&self) -> usize {
821 self.token_ids.len()
822 }
823
824 pub fn clear(&mut self) {
825 self.token_ids.clear();
826 self.prefix_offset = 0;
827 self.read_offset = 0;
828 }
829
830 pub fn append_text(&mut self, input: &str) -> Result<()> {
831 let encoding = self.tokenizer.encode(input)?;
836 self.token_ids.extend(encoding.token_ids());
837 Ok(())
838 }
839
840 pub fn append_token_id(&mut self, token_id: TokenIdType) -> Result<String> {
844 self.token_ids.push(token_id);
845 let prefix_text: String = self
848 .tokenizer
849 .decode(&self.token_ids[self.prefix_offset..self.read_offset], false)?
850 .into();
851
852 let new_result = self
853 .tokenizer
854 .decode(&self.token_ids[self.prefix_offset..], false)?;
855
856 let new_text = new_result.as_str();
857
858 let mut prefix_text_len = prefix_text.len();
862 while !new_text.is_char_boundary(prefix_text_len) && prefix_text_len > 0 {
863 prefix_text_len -= 1;
864 }
865 let prefix_text_len = prefix_text_len;
866
867 if new_text.len() > prefix_text.len() {
868 if new_result.is_partial() {
869 return Ok("".to_string());
870 } else {
871 let new_text = new_text[prefix_text_len..]
873 .to_string()
874 .replace('\u{FFFD}', "");
875 self.prefix_offset = self.read_offset;
876 self.read_offset = self.token_ids.len();
877 return Ok(new_text);
878 }
879 }
880
881 Ok("".to_string())
882 }
883
884 pub fn tokenizer(&self) -> Tokenizer {
885 self.tokenizer.clone()
886 }
887
888 pub fn token_ids(&self) -> &[TokenIdType] {
889 &self.token_ids
890 }
891
892 pub fn text(&self) -> Result<String> {
893 Ok(self.tokenizer.decode(&self.token_ids, false)?.into())
897 }
898}
899
900pub enum SequenceDecoderOutput {
903 Text(String),
905
906 Held,
909
910 Stopped,
913
914 StoppedWithText(String),
918}
919
920#[derive(Debug)]
926pub struct StopSequenceDecoder {
927 sequence: Sequence,
929
930 stop_token_ids_visible: Vec<TokenIdType>,
933
934 stop_token_ids_hidden: Vec<TokenIdType>,
937
938 #[allow(dead_code)]
941 stop_sequences_visible: Vec<String>,
942
943 stop_sequences_hidden: Vec<String>,
946
947 stopped: bool,
950
951 state: String,
954}
955
956impl StopSequenceDecoder {
957 pub fn builder(tokenizer: Tokenizer) -> StopSequenceDecoderBuilder {
959 StopSequenceDecoderBuilder::new(tokenizer)
960 }
961
962 pub fn append_token_id(&mut self, token_id: TokenIdType) -> Result<SequenceDecoderOutput> {
964 if self.stopped {
965 return Err(Error::msg("Decoder is stopped"));
966 }
967
968 let text = self.sequence.append_token_id(token_id)?;
970
971 self.state.push_str(text.as_str());
973
974 let mut stop: bool = false;
975 let mut visible: bool = false;
976
977 if self.stop_token_ids_visible.contains(&token_id) {
978 stop = true;
979 visible = true;
980 }
981
982 if self.stop_token_ids_hidden.contains(&token_id) {
983 stop = true;
984 visible = false;
985 }
986
987 if stop {
988 self.stopped = true;
989 let state = std::mem::take(&mut self.state);
990 if visible {
991 return Ok(SequenceDecoderOutput::StoppedWithText(state));
992 }
993 return Ok(SequenceDecoderOutput::Stopped);
994 }
995
996 for stop_sequence in self.stop_sequences_hidden.iter() {
998 if stop_sequence.starts_with(&self.state) {
999 if stop_sequence == &self.state {
1000 self.stopped = true;
1002 return Ok(SequenceDecoderOutput::Stopped);
1003 } else {
1004 return Ok(SequenceDecoderOutput::Held);
1005 }
1006 }
1007 }
1008
1009 let state = std::mem::take(&mut self.state);
1010 Ok(SequenceDecoderOutput::Text(state))
1011 }
1012
1013 pub fn is_empty(&self) -> bool {
1014 self.sequence.token_ids.is_empty()
1015 }
1016
1017 pub fn len(&self) -> usize {
1018 self.sequence.token_ids.len()
1019 }
1020
1021 pub fn is_complete(&self) -> bool {
1022 self.stopped
1023 }
1024
1025 pub fn close(&mut self) {
1026 self.stopped = true;
1027 }
1028}
1029
1030pub struct StopSequenceDecoderBuilder {
1031 tokenizer: Tokenizer,
1032 stop_token_ids_visible: Vec<TokenIdType>,
1033 stop_token_ids_hidden: Vec<TokenIdType>,
1034 stop_sequences_visible: Vec<String>,
1035 stop_sequences_hidden: Vec<String>,
1036}
1037
1038impl StopSequenceDecoderBuilder {
1039 pub fn new(tokenizer: Tokenizer) -> Self {
1040 Self {
1041 tokenizer,
1042 stop_token_ids_visible: Vec::new(),
1043 stop_token_ids_hidden: Vec::new(),
1044 stop_sequences_visible: Vec::new(),
1045 stop_sequences_hidden: Vec::new(),
1046 }
1047 }
1048
1049 pub fn add_stop_token_id_visible(mut self, token_id: TokenIdType) -> Self {
1051 self.stop_token_ids_visible.push(token_id);
1052 self
1053 }
1054
1055 pub fn add_stop_token_ids_visible(mut self, token_ids: &[TokenIdType]) -> Self {
1058 self.stop_token_ids_visible.extend(token_ids);
1059 self
1060 }
1061
1062 pub fn add_stop_token_id_hidden(mut self, token_id: TokenIdType) -> Self {
1064 self.stop_token_ids_hidden.push(token_id);
1065 self
1066 }
1067
1068 pub fn add_stop_token_ids_hidden(mut self, token_ids: &[TokenIdType]) -> Self {
1071 self.stop_token_ids_hidden.extend(token_ids);
1072 self
1073 }
1074
1075 pub fn add_stop_sequence_visible(mut self, text: &str) -> Self {
1076 self.stop_sequences_visible.push(text.to_string());
1077 self
1078 }
1079
1080 pub fn add_stop_sequences_visible(mut self, strings: &[&str]) -> Self {
1081 self.stop_sequences_visible
1082 .extend(strings.iter().map(|text| text.to_string()));
1083 self
1084 }
1085
1086 pub fn add_stop_sequence_hidden(mut self, text: &str) -> Self {
1087 self.stop_sequences_hidden.push(text.to_string());
1088 self
1089 }
1090
1091 pub fn add_stop_sequences_hidden(mut self, strings: &[&str]) -> Self {
1092 self.stop_sequences_hidden
1093 .extend(strings.iter().map(|text| text.to_string()));
1094 self
1095 }
1096
1097 pub fn build(self) -> Result<StopSequenceDecoder> {
1098 Ok(StopSequenceDecoder {
1099 sequence: Sequence::new(self.tokenizer.clone()),
1100 stop_token_ids_visible: self.stop_token_ids_visible,
1101 stop_token_ids_hidden: self.stop_token_ids_hidden,
1102 stop_sequences_visible: self.stop_sequences_visible,
1103 stop_sequences_hidden: self.stop_sequences_hidden,
1104 stopped: false,
1105 state: String::new(),
1106 })
1107 }
1108}