dynamo_tokenizers/
lib.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4pub mod fastokens;
5pub mod hf;
6pub mod tiktoken;
7
8// TODO: Add tokenizer benchmarks
9// TODO: Enable README.md as a module doc
10// #[doc = include_str!("../README.md")]
11
12use std::hash::{DefaultHasher, Hash, Hasher};
13use std::sync::Arc;
14use std::{fs::File, io::BufReader, ops::Deref, path::Path};
15
16use anyhow::Context as _;
17pub use anyhow::{Error, Result};
18
19pub use fastokens::FastTokenizer;
20pub use hf::HuggingFaceTokenizer;
21pub use tiktoken::TikTokenTokenizer;
22pub use traits::DecodeResult;
23
24pub type TokenIdType = u32;
25
26/// Represents the type of tokenizer being used
27#[derive(Debug)]
28pub enum TokenizerType {
29    HuggingFace(String),
30    TikToken(String),
31}
32
33/// character offsets in the original text
34pub type Offsets = (usize, usize);
35
36/// Contains the results of tokenizing text: token IDs, string tokens, and their spans
37#[derive(Debug, Clone)]
38pub enum Encoding {
39    /// Hugging Face
40    Hf(Box<tokenizers::tokenizer::Encoding>),
41    /// Sentence Piece
42    Sp(Vec<TokenIdType>),
43}
44
45impl Encoding {
46    pub fn token_ids(&self) -> &[u32] {
47        match self {
48            Encoding::Hf(inner) => inner.get_ids(),
49            Encoding::Sp(inner) => inner,
50        }
51    }
52}
53
54impl Hash for Encoding {
55    fn hash<H: Hasher>(&self, state: &mut H) {
56        self.token_ids().hash(state);
57    }
58}
59
60pub mod traits {
61    use super::*;
62
63    pub trait Encoder: Send + Sync {
64        fn encode(&self, input: &str) -> Result<Encoding>;
65        fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>>;
66    }
67
68    /// Result of decoding token IDs to text.
69    ///
70    /// Distinguishes between fully valid UTF-8 output and output that contains
71    /// trailing incomplete multi-byte sequences (represented as U+FFFD).
72    /// This lets callers like `DecodeStream::step()` decide whether to emit or
73    /// buffer without resorting to hardcoded replacement-character string checks.
74    #[derive(Debug, Clone, PartialEq, Eq, strum::EnumIs)]
75    pub enum DecodeResult {
76        /// No trailing incomplete multi-byte sequences (text does not end with U+FFFD).
77        /// Note: the string may still contain *interior* U+FFFD characters from
78        /// mid-stream invalid byte sequences; only trailing status is tracked here.
79        Complete(String),
80        /// The decoded string ends with U+FFFD, indicating incomplete trailing
81        /// multi-byte bytes that may be completed by subsequent tokens.
82        Partial(String),
83    }
84
85    impl DecodeResult {
86        /// Returns a reference to the inner string.
87        pub fn as_str(&self) -> &str {
88            match self {
89                DecodeResult::Complete(s) | DecodeResult::Partial(s) => s,
90            }
91        }
92
93        /// Construct from a decoded string: `Partial` if it ends with U+FFFD, else `Complete`.
94        pub fn from_decoded(text: String) -> Self {
95            if text.ends_with('\u{FFFD}') {
96                DecodeResult::Partial(text)
97            } else {
98                DecodeResult::Complete(text)
99            }
100        }
101    }
102
103    impl From<String> for DecodeResult {
104        fn from(text: String) -> Self {
105            DecodeResult::from_decoded(text)
106        }
107    }
108
109    impl From<DecodeResult> for String {
110        fn from(result: DecodeResult) -> Self {
111            match result {
112                DecodeResult::Complete(s) | DecodeResult::Partial(s) => s,
113            }
114        }
115    }
116
117    /// Implementations must ensure that partial multi-byte sequences produce U+FFFD
118    /// (`\u{FFFD}`) in the output rather than returning `Err`. This is commonly achieved
119    /// via `String::from_utf8_lossy` (tiktoken) or library-internal byte-fallback handling
120    /// (HuggingFace). `DecodeStream::step()` relies on `DecodeResult::Partial` to detect
121    /// incomplete sequences and buffer tokens until the full character arrives.
122    pub trait Decoder: Send + Sync {
123        fn decode(
124            &self,
125            token_ids: &[TokenIdType],
126            skip_special_tokens: bool,
127        ) -> Result<DecodeResult>;
128    }
129
130    pub trait Tokenizer: Encoder + Decoder {
131        // fn get_vocab_size(&self) -> usize;
132        // fn make_unique_clone(&self) -> Box<dyn Tokenizer>;
133    }
134}
135
136pub fn file_json_field<T: serde::de::DeserializeOwned>(
137    json_file_path: &Path,
138    field_name: &str,
139) -> anyhow::Result<T> {
140    let file = File::open(json_file_path)
141        .with_context(|| format!("Failed to open file: {:?}", json_file_path))?;
142    let reader = BufReader::new(file);
143
144    let json_data: serde_json::Value = serde_json::from_reader(reader)
145        .with_context(|| format!("Failed to parse JSON from file: {:?}", json_file_path))?;
146
147    let map = json_data.as_object().ok_or_else(|| {
148        anyhow::anyhow!("JSON root is not an object in file: {:?}", json_file_path)
149    })?;
150
151    let field_value = map.get(field_name).ok_or_else(|| {
152        anyhow::anyhow!(
153            "Field '{}' not found in JSON file: {:?}",
154            field_name,
155            json_file_path
156        )
157    })?;
158
159    serde_json::from_value(field_value.clone()).with_context(|| {
160        format!(
161            "Failed to deserialize field '{}' (value: {:?}) to the expected type from file: {:?}",
162            field_name, field_value, json_file_path
163        )
164    })
165}
166
167pub fn log_json_err(filename: &str, json: &str, err: &serde_json::Error) {
168    const ERROR_PREFIX: &str = ">>     ";
169
170    if !(err.is_syntax() || err.is_data()) {
171        return;
172    }
173
174    let line = err.line().saturating_sub(1);
175    let column = err.column().saturating_sub(1);
176
177    let json_lines: Vec<&str> = json.lines().collect();
178    if json_lines.is_empty() {
179        tracing::error!("JSON parsing error in {filename}: File is empty.");
180        return;
181    }
182
183    let start_index = line.saturating_sub(2);
184    let end_index = line.saturating_add(3).min(json_lines.len());
185
186    let mut context_lines: Vec<String> = (start_index..end_index)
187        .map(|i| {
188            if i == line {
189                format!("{ERROR_PREFIX}{}", json_lines[i])
190            } else {
191                format!("{:06} {}", i + 1, json_lines[i])
192            }
193        })
194        .collect();
195
196    let col_indicator = "_".to_string().repeat(column + ERROR_PREFIX.len()) + "^";
197    let error_in_context_idx = line - start_index;
198    if error_in_context_idx < context_lines.len() {
199        context_lines.insert(error_in_context_idx + 1, col_indicator);
200    }
201
202    tracing::error!(
203        "JSON parsing error in {filename}: Line {}, column {}:\n{}",
204        err.line(),
205        err.column(),
206        context_lines.join("\n")
207    );
208}
209
210impl Encoding {
211    pub fn get_hash(&self) -> u64 {
212        let mut hasher = DefaultHasher::new();
213        self.hash(&mut hasher);
214        hasher.finish()
215    }
216}
217
218/// Main tokenizer wrapper that provides a unified interface for different tokenizer implementations
219#[derive(Clone)]
220pub struct Tokenizer(Arc<dyn traits::Tokenizer>);
221
222impl Tokenizer {
223    pub fn from_file(file_path: &str) -> Result<Tokenizer> {
224        Ok(Tokenizer(create_tokenizer_from_file(file_path)?))
225    }
226
227    /// Create a stateful sequence object for decoding token_ids into text
228    pub fn decode_stream(
229        &self,
230        prompt_token_ids: &[TokenIdType],
231        skip_special_tokens: bool,
232    ) -> DecodeStream {
233        DecodeStream::new(self.0.clone(), prompt_token_ids, skip_special_tokens)
234    }
235}
236
237impl Deref for Tokenizer {
238    type Target = Arc<dyn traits::Tokenizer>;
239
240    fn deref(&self) -> &Self::Target {
241        &self.0
242    }
243}
244
245impl From<Arc<dyn traits::Tokenizer>> for Tokenizer {
246    fn from(tokenizer: Arc<dyn traits::Tokenizer>) -> Self {
247        Tokenizer(tokenizer)
248    }
249}
250
251impl<T> From<Arc<T>> for Tokenizer
252where
253    T: traits::Tokenizer + 'static, // 'static is required to ensure T can be safely put into an Arc
254{
255    fn from(tokenizer: Arc<T>) -> Self {
256        Tokenizer(tokenizer)
257    }
258}
259
260/// Create a tokenizer from a file path to a tokenizer file.
261/// The file extension is used to determine the tokenizer type.
262/// Supported file types are:
263/// - json: HuggingFace tokenizer
264/// - model, tiktoken: tiktoken BPE tokenizer (requires `config.json` with a supported
265///   `model_type` in the same directory; currently: kimi, kimi_k2, kimi_k25)
266pub fn create_tokenizer_from_file(file_path: &str) -> Result<Arc<dyn traits::Tokenizer>> {
267    let path = Path::new(file_path);
268    let extension = path
269        .extension()
270        .and_then(std::ffi::OsStr::to_str)
271        .ok_or_else(|| Error::msg("Failed to read file extension".to_string()))?;
272
273    match extension {
274        "json" => {
275            let tokenizer = HuggingFaceTokenizer::from_file(file_path)?;
276            Ok(Arc::new(tokenizer))
277        }
278        "model" | "tiktoken" => {
279            let tokenizer = TikTokenTokenizer::from_file_auto(file_path)?;
280            Ok(Arc::new(tokenizer))
281        }
282        _ => Err(Error::msg(format!(
283            "Unsupported tokenizer file type: .{extension}"
284        ))),
285    }
286}
287
288// With incremental detokenization, we need to consider the final context tokens when handling the initial decode tokens.
289// This is the initial offset from the end of the context that we start decoding from.
290// Both Huggingface TGI and vLLM use this same value.
291// See: https://github.com/huggingface/text-generation-inference/blob/24c2bff65924801ddf90fa24fcc72752d4f45538/server/text_generation_server/models/mamba.py#L169
292// and https://github.com/vllm-project/vllm/blob/da2705198fa19030a25d0bea437f7be6547d47d4/vllm/transformers_utils/detokenizer_utils.py#L51
293const INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET: usize = 5;
294
295/// DecodeStream will keep the state necessary to produce individual chunks of
296/// strings given an input stream of token_ids.
297///
298/// This is necessary because decoding in general cannot achieve that since strings
299/// depend on surrounding ids to provide a valid string. Typically stripping extra spaces.
300pub struct DecodeStream {
301    /// The tokenizer used to decode token_ids
302    tokenizer: Arc<dyn traits::Tokenizer>,
303
304    skip_special_tokens: bool,
305    /// A temporary buffer of the necessary token_ids needed
306    /// to produce valid string chunks.
307    /// This typically contains 3 parts:
308    ///  - read
309    ///  - prefix
310    ///  - rest
311    ///
312    /// Read is the bit necessary to surround the prefix
313    /// so decoding the whole ids produces a valid prefix.
314    /// Prefix is the previously produced string, kept around to trim off of
315    /// the next valid chunk
316    all_token_ids: Vec<u32>,
317
318    prefix_offset: usize,
319
320    read_offset: usize,
321}
322
323impl DecodeStream {
324    pub fn new(
325        tokenizer: Arc<dyn traits::Tokenizer>,
326        prompt_token_ids: &[TokenIdType],
327        skip_special_tokens: bool,
328    ) -> Self {
329        let num_input_tokens = prompt_token_ids.len();
330        let prompt_token_ids = prompt_token_ids.to_vec();
331        Self {
332            tokenizer,
333            skip_special_tokens,
334            all_token_ids: prompt_token_ids,
335            prefix_offset: num_input_tokens
336                .saturating_sub(INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET),
337            read_offset: num_input_tokens,
338        }
339    }
340
341    /// Step appends a token_id to the internal state and tries to produce a text chunk.
342    ///
343    /// Implementation directly copied from Huggingface's TGI:
344    /// https://github.com/huggingface/text-generation-inference/blob/24c2bff65924801ddf90fa24fcc72752d4f45538/server/text_generation_server/models/model.py#L144
345    ///
346    /// Returning `None` means the given id is not enough to produce a chunk.
347    /// This typically happens with `byte_fallback` options where some tokens do not
348    /// represent valid UTF-8, and only follow-up token_ids will help produce
349    /// a valid chunk.
350    pub fn step(&mut self, id: u32) -> Result<Option<String>> {
351        self.all_token_ids.push(id);
352
353        let prefix_text: String = self
354            .tokenizer
355            .decode(
356                &self.all_token_ids[self.prefix_offset..self.read_offset],
357                self.skip_special_tokens,
358            )?
359            .into();
360
361        let new_result = self.tokenizer.decode(
362            &self.all_token_ids[self.prefix_offset..],
363            self.skip_special_tokens,
364        )?;
365
366        let new_text = new_result.as_str();
367        if new_text.len() > prefix_text.len() && !new_result.is_partial() {
368            let emitted = new_text[prefix_text.len()..].to_string();
369
370            self.prefix_offset = self.read_offset;
371            self.read_offset = self.all_token_ids.len();
372
373            Ok(Some(emitted))
374        } else {
375            Ok(None)
376        }
377    }
378}
379
380/// Maintains state for an ongoing sequence of tokens and their decoded text
381pub struct Sequence {
382    /// Encodes text -> token_ids
383    tokenizer: Tokenizer,
384
385    /// The current sequence of token ids
386    token_ids: Vec<TokenIdType>,
387
388    /// The position in the current sequence the last decoded token completed
389    prefix_offset: usize,
390
391    /// Current position in the sequence
392    read_offset: usize,
393}
394
395impl std::fmt::Debug for Sequence {
396    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
397        f.debug_struct("Sequence")
398            .field("tokenizer", &"Arc<dyn Tokenizer>")
399            .field(
400                "token_ids",
401                &format_args!("{}", {
402                    let token_ids = self.token_ids();
403                    if token_ids.len() <= 20 {
404                        format!("{:?}", token_ids)
405                    } else {
406                        let first_ten = &token_ids[..10];
407                        let last_ten = &token_ids[token_ids.len() - 10..];
408                        format!("{:?} ... {:?}", first_ten, last_ten)
409                    }
410                }),
411            )
412            .field("prefix_offset", &self.prefix_offset)
413            .field("read_offset", &self.read_offset)
414            .field("token count", &self.token_ids.len())
415            .finish()
416    }
417}
418
419impl Sequence {
420    pub fn new(tokenizer: Tokenizer) -> Self {
421        Self {
422            tokenizer,
423            token_ids: Vec::new(),
424            prefix_offset: 0,
425            read_offset: 0,
426        }
427    }
428
429    pub fn is_empty(&self) -> bool {
430        self.token_ids.is_empty()
431    }
432
433    pub fn len(&self) -> usize {
434        self.token_ids.len()
435    }
436
437    pub fn clear(&mut self) {
438        self.token_ids.clear();
439        self.prefix_offset = 0;
440        self.read_offset = 0;
441    }
442
443    pub fn append_text(&mut self, input: &str) -> Result<()> {
444        // let tokenizer = self.tokenizer.read().map_err(|err| {
445        //     Error::msg(format!("Failed to acquire read lock on tokenizer: {}", err))
446        // })?;
447
448        let encoding = self.tokenizer.encode(input)?;
449        self.token_ids.extend(encoding.token_ids());
450        Ok(())
451    }
452
453    // Based on
454    // https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
455    // under Apache 2.0 license
456    pub fn append_token_id(&mut self, token_id: TokenIdType) -> Result<String> {
457        self.token_ids.push(token_id);
458        // log::trace!("pushed token_id: {}", token_id);
459
460        let prefix_text: String = self
461            .tokenizer
462            .decode(&self.token_ids[self.prefix_offset..self.read_offset], false)?
463            .into();
464
465        let new_result = self
466            .tokenizer
467            .decode(&self.token_ids[self.prefix_offset..], false)?;
468
469        let new_text = new_result.as_str();
470
471        // if the end character of the previous returned sequence is a multi-byte character
472        // then we can not split the text on that byte offset, so we roll back to the byte offset
473        // of the start of that character
474        let mut prefix_text_len = prefix_text.len();
475        while !new_text.is_char_boundary(prefix_text_len) && prefix_text_len > 0 {
476            prefix_text_len -= 1;
477        }
478        let prefix_text_len = prefix_text_len;
479
480        if new_text.len() > prefix_text.len() {
481            if new_result.is_partial() {
482                return Ok("".to_string());
483            } else {
484                // shift and update the state
485                let new_text = new_text[prefix_text_len..]
486                    .to_string()
487                    .replace('\u{FFFD}', "");
488                self.prefix_offset = self.read_offset;
489                self.read_offset = self.token_ids.len();
490                return Ok(new_text);
491            }
492        }
493
494        Ok("".to_string())
495    }
496
497    pub fn tokenizer(&self) -> Tokenizer {
498        self.tokenizer.clone()
499    }
500
501    pub fn token_ids(&self) -> &[TokenIdType] {
502        &self.token_ids
503    }
504
505    pub fn text(&self) -> Result<String> {
506        // let tokenizer = self.tokenizer.read().map_err(|err| {
507        //     Error::msg(format!("Failed to acquire read lock on tokenizer: {}", err))
508        // })?;
509        Ok(self.tokenizer.decode(&self.token_ids, false)?.into())
510    }
511}
512
513/// The output conditions/values of a SequenceDecoder::add_token_id operation.
514/// Result of decoding a token, indicating whether text was produced or a stop condition was met
515pub enum SequenceDecoderOutput {
516    /// The text for the appended token_id
517    Text(String),
518
519    /// A sequence of token_ids has been partially matched a stop sequence, so the text is held
520    /// until either a match or a divergence
521    Held,
522
523    /// Indicates that a stop sequence has been matched and the decoder is stopped.
524    /// Subsequent calls to append_token_id will return an error
525    Stopped,
526
527    /// Indicates that a stop token_id has been matched and the decoder is stopped.
528    /// Subsequent calls to append_token_id will return an error
529    /// The text for the stop token_id is returned
530    StoppedWithText(String),
531}
532
533/// A Sequence for decoding a stream of token ids into text and detecting stop sequences.
534/// A stop sequence is either a matching token_id or a sequence of texts/strings which match.
535/// Matches happen first at the token-level, then at the sequence-level. Hidden takes precedence
536/// over visible. For example, if you put the same token_id in both `stop_token_ids_visible` and
537/// `stop_token_ids_hidden`, the token_id will be treated as hidden.
538#[derive(Debug)]
539pub struct StopSequenceDecoder {
540    // The current sequence of token ids
541    sequence: Sequence,
542
543    // Stop Tokens - the presence of any one of these should trigger a stop
544    // If found, the text for the matched token will be returned
545    stop_token_ids_visible: Vec<TokenIdType>,
546
547    // Stop Tokens - the presence of any one of these should trigger a stop
548    // If found, the text for the matched token will NOT be returned
549    stop_token_ids_hidden: Vec<TokenIdType>,
550
551    // Stop Words - the presence of any one of these should trigger a stop
552    // If found, the text for the matched token will be returned
553    #[allow(dead_code)]
554    stop_sequences_visible: Vec<String>,
555
556    // Stop Words - the presence of any one of these should trigger a stop
557    // If found, the text for the matched token will NOT be returned
558    stop_sequences_hidden: Vec<String>,
559
560    // If the decoder has observed and returned a stop SequenceDecoderOutput,
561    // futhur calls to append_token_id will return an error
562    stopped: bool,
563
564    // text jail - if a partial stop sequence is being observed, we hold/jail the text
565    // until either the stop sequence is matched or the sequence is reset by a divergence
566    state: String,
567}
568
569impl StopSequenceDecoder {
570    /// Builder object for configurating a StopSequenceDecoder
571    pub fn builder(tokenizer: Tokenizer) -> StopSequenceDecoderBuilder {
572        StopSequenceDecoderBuilder::new(tokenizer)
573    }
574
575    /// Add a token_id to the sequence and return the SequenceDecoderOutput
576    pub fn append_token_id(&mut self, token_id: TokenIdType) -> Result<SequenceDecoderOutput> {
577        if self.stopped {
578            return Err(Error::msg("Decoder is stopped"));
579        }
580
581        // update the sequence
582        let text = self.sequence.append_token_id(token_id)?;
583
584        // append the text to the state
585        self.state.push_str(text.as_str());
586
587        let mut stop: bool = false;
588        let mut visible: bool = false;
589
590        if self.stop_token_ids_visible.contains(&token_id) {
591            stop = true;
592            visible = true;
593        }
594
595        if self.stop_token_ids_hidden.contains(&token_id) {
596            stop = true;
597            visible = false;
598        }
599
600        if stop {
601            self.stopped = true;
602            let state = std::mem::take(&mut self.state);
603            if visible {
604                return Ok(SequenceDecoderOutput::StoppedWithText(state));
605            }
606            return Ok(SequenceDecoderOutput::Stopped);
607        }
608
609        // determine if state matches any of the stop sequences
610        for stop_sequence in self.stop_sequences_hidden.iter() {
611            if stop_sequence.starts_with(&self.state) {
612                if stop_sequence == &self.state {
613                    // on matched stop sequence, we do NOT return the jailed stop sequence
614                    self.stopped = true;
615                    return Ok(SequenceDecoderOutput::Stopped);
616                } else {
617                    return Ok(SequenceDecoderOutput::Held);
618                }
619            }
620        }
621
622        let state = std::mem::take(&mut self.state);
623        Ok(SequenceDecoderOutput::Text(state))
624    }
625
626    pub fn is_empty(&self) -> bool {
627        self.sequence.token_ids.is_empty()
628    }
629
630    pub fn len(&self) -> usize {
631        self.sequence.token_ids.len()
632    }
633
634    pub fn is_complete(&self) -> bool {
635        self.stopped
636    }
637
638    pub fn close(&mut self) {
639        self.stopped = true;
640    }
641}
642
643pub struct StopSequenceDecoderBuilder {
644    tokenizer: Tokenizer,
645    stop_token_ids_visible: Vec<TokenIdType>,
646    stop_token_ids_hidden: Vec<TokenIdType>,
647    stop_sequences_visible: Vec<String>,
648    stop_sequences_hidden: Vec<String>,
649}
650
651impl StopSequenceDecoderBuilder {
652    pub fn new(tokenizer: Tokenizer) -> Self {
653        Self {
654            tokenizer,
655            stop_token_ids_visible: Vec::new(),
656            stop_token_ids_hidden: Vec::new(),
657            stop_sequences_visible: Vec::new(),
658            stop_sequences_hidden: Vec::new(),
659        }
660    }
661
662    /// Adds a visible stop token id to the StopSequenceDecoder
663    pub fn add_stop_token_id_visible(mut self, token_id: TokenIdType) -> Self {
664        self.stop_token_ids_visible.push(token_id);
665        self
666    }
667
668    /// Adds a list of visible stop token ids to the StopSequenceDecoder
669    /// Each token_id is added as for an individual match
670    pub fn add_stop_token_ids_visible(mut self, token_ids: &[TokenIdType]) -> Self {
671        self.stop_token_ids_visible.extend(token_ids);
672        self
673    }
674
675    /// Adds a hidden stop token id to the StopSequenceDecoder
676    pub fn add_stop_token_id_hidden(mut self, token_id: TokenIdType) -> Self {
677        self.stop_token_ids_hidden.push(token_id);
678        self
679    }
680
681    /// Adds a list of hidden stop token ids to the StopSequenceDecoder
682    /// Each token_id is added as for an individual match
683    pub fn add_stop_token_ids_hidden(mut self, token_ids: &[TokenIdType]) -> Self {
684        self.stop_token_ids_hidden.extend(token_ids);
685        self
686    }
687
688    pub fn add_stop_sequence_visible(mut self, text: &str) -> Self {
689        self.stop_sequences_visible.push(text.to_string());
690        self
691    }
692
693    pub fn add_stop_sequences_visible(mut self, strings: &[&str]) -> Self {
694        self.stop_sequences_visible
695            .extend(strings.iter().map(|text| text.to_string()));
696        self
697    }
698
699    pub fn add_stop_sequence_hidden(mut self, text: &str) -> Self {
700        self.stop_sequences_hidden.push(text.to_string());
701        self
702    }
703
704    pub fn add_stop_sequences_hidden(mut self, strings: &[&str]) -> Self {
705        self.stop_sequences_hidden
706            .extend(strings.iter().map(|text| text.to_string()));
707        self
708    }
709
710    pub fn build(self) -> Result<StopSequenceDecoder> {
711        Ok(StopSequenceDecoder {
712            sequence: Sequence::new(self.tokenizer.clone()),
713            stop_token_ids_visible: self.stop_token_ids_visible,
714            stop_token_ids_hidden: self.stop_token_ids_hidden,
715            stop_sequences_visible: self.stop_sequences_visible,
716            stop_sequences_hidden: self.stop_sequences_hidden,
717            stopped: false,
718            state: String::new(),
719        })
720    }
721}
dynamo_tokenizers/lib.rs

dynamo_tokenizers/
lib.rs
