dynamo_tokenizers/
lib.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4pub mod cache;
5pub mod fastokens;
6pub mod hf;
7pub mod tiktoken;
8
9// TODO: Add tokenizer benchmarks
10// TODO: Enable README.md as a module doc
11// #[doc = include_str!("../README.md")]
12
13use std::hash::{DefaultHasher, Hash, Hasher};
14use std::sync::Arc;
15use std::{fs::File, io::BufReader, ops::Deref, path::Path};
16
17use anyhow::Context as _;
18pub use anyhow::{Error, Result};
19
20pub use cache::{CachedTokenizer, L1CacheStats};
21pub use fastokens::FastTokenizer;
22pub use hf::HuggingFaceTokenizer;
23pub use tiktoken::TikTokenTokenizer;
24pub use traits::DecodeResult;
25
26pub type TokenIdType = u32;
27
28/// Represents the type of tokenizer being used
29#[derive(Debug)]
30pub enum TokenizerType {
31    HuggingFace(String),
32    TikToken(String),
33}
34
35/// character offsets in the original text
36pub type Offsets = (usize, usize);
37
38/// Contains the results of tokenizing text: token IDs, string tokens, and their spans
39#[derive(Debug, Clone)]
40pub enum Encoding {
41    /// Hugging Face
42    Hf(Box<tokenizers::tokenizer::Encoding>),
43    /// Sentence Piece
44    Sp(Vec<TokenIdType>),
45}
46
47impl Encoding {
48    pub fn token_ids(&self) -> &[u32] {
49        match self {
50            Encoding::Hf(inner) => inner.get_ids(),
51            Encoding::Sp(inner) => inner,
52        }
53    }
54}
55
56impl Hash for Encoding {
57    fn hash<H: Hasher>(&self, state: &mut H) {
58        self.token_ids().hash(state);
59    }
60}
61
62pub mod traits {
63    use super::*;
64
65    pub trait Encoder: Send + Sync {
66        fn encode(&self, input: &str) -> Result<Encoding>;
67        fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>>;
68    }
69
70    /// Result of decoding token IDs to text.
71    ///
72    /// Distinguishes between fully valid UTF-8 output and output that contains
73    /// trailing incomplete multi-byte sequences (represented as U+FFFD).
74    /// This lets callers like `DecodeStream::step()` decide whether to emit or
75    /// buffer without resorting to hardcoded replacement-character string checks.
76    #[derive(Debug, Clone, PartialEq, Eq, strum::EnumIs)]
77    pub enum DecodeResult {
78        /// No trailing incomplete multi-byte sequences (text does not end with U+FFFD).
79        /// Note: the string may still contain *interior* U+FFFD characters from
80        /// mid-stream invalid byte sequences; only trailing status is tracked here.
81        Complete(String),
82        /// The decoded string ends with U+FFFD, indicating incomplete trailing
83        /// multi-byte bytes that may be completed by subsequent tokens.
84        Partial(String),
85    }
86
87    impl DecodeResult {
88        /// Returns a reference to the inner string.
89        pub fn as_str(&self) -> &str {
90            match self {
91                DecodeResult::Complete(s) | DecodeResult::Partial(s) => s,
92            }
93        }
94
95        /// Construct from a decoded string: `Partial` if it ends with U+FFFD, else `Complete`.
96        pub fn from_decoded(text: String) -> Self {
97            if text.ends_with('\u{FFFD}') {
98                DecodeResult::Partial(text)
99            } else {
100                DecodeResult::Complete(text)
101            }
102        }
103    }
104
105    impl From<String> for DecodeResult {
106        fn from(text: String) -> Self {
107            DecodeResult::from_decoded(text)
108        }
109    }
110
111    impl From<DecodeResult> for String {
112        fn from(result: DecodeResult) -> Self {
113            match result {
114                DecodeResult::Complete(s) | DecodeResult::Partial(s) => s,
115            }
116        }
117    }
118
119    /// Implementations must ensure that partial multi-byte sequences produce U+FFFD
120    /// (`\u{FFFD}`) in the output rather than returning `Err`. This is commonly achieved
121    /// via `String::from_utf8_lossy` (tiktoken) or library-internal byte-fallback handling
122    /// (HuggingFace). `DecodeStream::step()` relies on `DecodeResult::Partial` to detect
123    /// incomplete sequences and buffer tokens until the full character arrives.
124    pub trait Decoder: Send + Sync {
125        fn decode(
126            &self,
127            token_ids: &[TokenIdType],
128            skip_special_tokens: bool,
129        ) -> Result<DecodeResult>;
130    }
131
132    pub trait Tokenizer: Encoder + Decoder {
133        // fn get_vocab_size(&self) -> usize;
134        // fn make_unique_clone(&self) -> Box<dyn Tokenizer>;
135    }
136}
137
138pub fn file_json_field<T: serde::de::DeserializeOwned>(
139    json_file_path: &Path,
140    field_name: &str,
141) -> anyhow::Result<T> {
142    let file = File::open(json_file_path)
143        .with_context(|| format!("Failed to open file: {:?}", json_file_path))?;
144    let reader = BufReader::new(file);
145
146    let json_data: serde_json::Value = serde_json::from_reader(reader)
147        .with_context(|| format!("Failed to parse JSON from file: {:?}", json_file_path))?;
148
149    let map = json_data.as_object().ok_or_else(|| {
150        anyhow::anyhow!("JSON root is not an object in file: {:?}", json_file_path)
151    })?;
152
153    let field_value = map.get(field_name).ok_or_else(|| {
154        anyhow::anyhow!(
155            "Field '{}' not found in JSON file: {:?}",
156            field_name,
157            json_file_path
158        )
159    })?;
160
161    serde_json::from_value(field_value.clone()).with_context(|| {
162        format!(
163            "Failed to deserialize field '{}' (value: {:?}) to the expected type from file: {:?}",
164            field_name, field_value, json_file_path
165        )
166    })
167}
168
169pub fn log_json_err(filename: &str, json: &str, err: &serde_json::Error) {
170    const ERROR_PREFIX: &str = ">>     ";
171
172    if !(err.is_syntax() || err.is_data()) {
173        return;
174    }
175
176    let line = err.line().saturating_sub(1);
177    let column = err.column().saturating_sub(1);
178
179    let json_lines: Vec<&str> = json.lines().collect();
180    if json_lines.is_empty() {
181        tracing::error!("JSON parsing error in {filename}: File is empty.");
182        return;
183    }
184
185    let start_index = line.saturating_sub(2);
186    let end_index = line.saturating_add(3).min(json_lines.len());
187
188    let mut context_lines: Vec<String> = (start_index..end_index)
189        .map(|i| {
190            if i == line {
191                format!("{ERROR_PREFIX}{}", json_lines[i])
192            } else {
193                format!("{:06} {}", i + 1, json_lines[i])
194            }
195        })
196        .collect();
197
198    let col_indicator = "_".to_string().repeat(column + ERROR_PREFIX.len()) + "^";
199    let error_in_context_idx = line - start_index;
200    if error_in_context_idx < context_lines.len() {
201        context_lines.insert(error_in_context_idx + 1, col_indicator);
202    }
203
204    tracing::error!(
205        "JSON parsing error in {filename}: Line {}, column {}:\n{}",
206        err.line(),
207        err.column(),
208        context_lines.join("\n")
209    );
210}
211
212impl Encoding {
213    pub fn get_hash(&self) -> u64 {
214        let mut hasher = DefaultHasher::new();
215        self.hash(&mut hasher);
216        hasher.finish()
217    }
218}
219
220/// Main tokenizer wrapper that provides a unified interface for different tokenizer implementations
221#[derive(Clone)]
222pub struct Tokenizer(Arc<dyn traits::Tokenizer>);
223
224impl Tokenizer {
225    pub fn from_file(file_path: &str) -> Result<Tokenizer> {
226        Ok(Tokenizer(create_tokenizer_from_file(file_path)?))
227    }
228
229    /// Create a stateful sequence object for decoding token_ids into text
230    pub fn decode_stream(
231        &self,
232        prompt_token_ids: &[TokenIdType],
233        skip_special_tokens: bool,
234    ) -> DecodeStream {
235        DecodeStream::new(self.0.clone(), prompt_token_ids, skip_special_tokens)
236    }
237}
238
239impl Deref for Tokenizer {
240    type Target = Arc<dyn traits::Tokenizer>;
241
242    fn deref(&self) -> &Self::Target {
243        &self.0
244    }
245}
246
247impl From<Arc<dyn traits::Tokenizer>> for Tokenizer {
248    fn from(tokenizer: Arc<dyn traits::Tokenizer>) -> Self {
249        Tokenizer(tokenizer)
250    }
251}
252
253impl<T> From<Arc<T>> for Tokenizer
254where
255    T: traits::Tokenizer + 'static, // 'static is required to ensure T can be safely put into an Arc
256{
257    fn from(tokenizer: Arc<T>) -> Self {
258        Tokenizer(tokenizer)
259    }
260}
261
262/// Create a tokenizer from a file path to a tokenizer file.
263/// The file extension is used to determine the tokenizer type.
264/// Supported file types are:
265/// - json: HuggingFace tokenizer
266/// - model, tiktoken: tiktoken BPE tokenizer (requires `config.json` with a supported
267///   `model_type` in the same directory; currently: kimi, kimi_k2, kimi_k25)
268pub fn create_tokenizer_from_file(file_path: &str) -> Result<Arc<dyn traits::Tokenizer>> {
269    let path = Path::new(file_path);
270    let extension = path
271        .extension()
272        .and_then(std::ffi::OsStr::to_str)
273        .ok_or_else(|| Error::msg("Failed to read file extension".to_string()))?;
274
275    match extension {
276        "json" => {
277            let tokenizer = HuggingFaceTokenizer::from_file(file_path)?;
278            Ok(Arc::new(tokenizer))
279        }
280        "model" | "tiktoken" => {
281            let tokenizer = TikTokenTokenizer::from_file_auto(file_path)?;
282            Ok(Arc::new(tokenizer))
283        }
284        _ => Err(Error::msg(format!(
285            "Unsupported tokenizer file type: .{extension}"
286        ))),
287    }
288}
289
290// With incremental detokenization, we need to consider the final context tokens when handling the initial decode tokens.
291// This is the initial offset from the end of the context that we start decoding from.
292// Both Huggingface TGI and vLLM use this same value.
293// See: https://github.com/huggingface/text-generation-inference/blob/24c2bff65924801ddf90fa24fcc72752d4f45538/server/text_generation_server/models/mamba.py#L169
294// and https://github.com/vllm-project/vllm/blob/da2705198fa19030a25d0bea437f7be6547d47d4/vllm/transformers_utils/detokenizer_utils.py#L51
295const INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET: usize = 5;
296
297/// DecodeStream will keep the state necessary to produce individual chunks of
298/// strings given an input stream of token_ids.
299///
300/// This is necessary because decoding in general cannot achieve that since strings
301/// depend on surrounding ids to provide a valid string. Typically stripping extra spaces.
302pub struct DecodeStream {
303    /// The tokenizer used to decode token_ids
304    tokenizer: Arc<dyn traits::Tokenizer>,
305
306    skip_special_tokens: bool,
307    /// A temporary buffer of the necessary token_ids needed
308    /// to produce valid string chunks.
309    /// This typically contains 3 parts:
310    ///  - read
311    ///  - prefix
312    ///  - rest
313    ///
314    /// Read is the bit necessary to surround the prefix
315    /// so decoding the whole ids produces a valid prefix.
316    /// Prefix is the previously produced string, kept around to trim off of
317    /// the next valid chunk
318    all_token_ids: Vec<u32>,
319
320    prefix_offset: usize,
321
322    read_offset: usize,
323}
324
325impl DecodeStream {
326    pub fn new(
327        tokenizer: Arc<dyn traits::Tokenizer>,
328        prompt_token_ids: &[TokenIdType],
329        skip_special_tokens: bool,
330    ) -> Self {
331        let num_input_tokens = prompt_token_ids.len();
332        let prompt_token_ids = prompt_token_ids.to_vec();
333        Self {
334            tokenizer,
335            skip_special_tokens,
336            all_token_ids: prompt_token_ids,
337            prefix_offset: num_input_tokens
338                .saturating_sub(INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET),
339            read_offset: num_input_tokens,
340        }
341    }
342
343    /// Step appends a token_id to the internal state and tries to produce a text chunk.
344    ///
345    /// Implementation directly copied from Huggingface's TGI:
346    /// https://github.com/huggingface/text-generation-inference/blob/24c2bff65924801ddf90fa24fcc72752d4f45538/server/text_generation_server/models/model.py#L144
347    ///
348    /// Returning `None` means the given id is not enough to produce a chunk.
349    /// This typically happens with `byte_fallback` options where some tokens do not
350    /// represent valid UTF-8, and only follow-up token_ids will help produce
351    /// a valid chunk.
352    pub fn step(&mut self, id: u32) -> Result<Option<String>> {
353        self.all_token_ids.push(id);
354
355        let prefix_text: String = self
356            .tokenizer
357            .decode(
358                &self.all_token_ids[self.prefix_offset..self.read_offset],
359                self.skip_special_tokens,
360            )?
361            .into();
362
363        let new_result = self.tokenizer.decode(
364            &self.all_token_ids[self.prefix_offset..],
365            self.skip_special_tokens,
366        )?;
367
368        let new_text = new_result.as_str();
369        if new_text.len() > prefix_text.len() && !new_result.is_partial() {
370            let emitted = new_text[prefix_text.len()..].to_string();
371
372            self.prefix_offset = self.read_offset;
373            self.read_offset = self.all_token_ids.len();
374
375            Ok(Some(emitted))
376        } else {
377            Ok(None)
378        }
379    }
380}
381
382/// Maintains state for an ongoing sequence of tokens and their decoded text
383pub struct Sequence {
384    /// Encodes text -> token_ids
385    tokenizer: Tokenizer,
386
387    /// The current sequence of token ids
388    token_ids: Vec<TokenIdType>,
389
390    /// The position in the current sequence the last decoded token completed
391    prefix_offset: usize,
392
393    /// Current position in the sequence
394    read_offset: usize,
395}
396
397impl std::fmt::Debug for Sequence {
398    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
399        f.debug_struct("Sequence")
400            .field("tokenizer", &"Arc<dyn Tokenizer>")
401            .field(
402                "token_ids",
403                &format_args!("{}", {
404                    let token_ids = self.token_ids();
405                    if token_ids.len() <= 20 {
406                        format!("{:?}", token_ids)
407                    } else {
408                        let first_ten = &token_ids[..10];
409                        let last_ten = &token_ids[token_ids.len() - 10..];
410                        format!("{:?} ... {:?}", first_ten, last_ten)
411                    }
412                }),
413            )
414            .field("prefix_offset", &self.prefix_offset)
415            .field("read_offset", &self.read_offset)
416            .field("token count", &self.token_ids.len())
417            .finish()
418    }
419}
420
421impl Sequence {
422    pub fn new(tokenizer: Tokenizer) -> Self {
423        Self {
424            tokenizer,
425            token_ids: Vec::new(),
426            prefix_offset: 0,
427            read_offset: 0,
428        }
429    }
430
431    pub fn is_empty(&self) -> bool {
432        self.token_ids.is_empty()
433    }
434
435    pub fn len(&self) -> usize {
436        self.token_ids.len()
437    }
438
439    pub fn clear(&mut self) {
440        self.token_ids.clear();
441        self.prefix_offset = 0;
442        self.read_offset = 0;
443    }
444
445    pub fn append_text(&mut self, input: &str) -> Result<()> {
446        // let tokenizer = self.tokenizer.read().map_err(|err| {
447        //     Error::msg(format!("Failed to acquire read lock on tokenizer: {}", err))
448        // })?;
449
450        let encoding = self.tokenizer.encode(input)?;
451        self.token_ids.extend(encoding.token_ids());
452        Ok(())
453    }
454
455    // Based on
456    // https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
457    // under Apache 2.0 license
458    pub fn append_token_id(&mut self, token_id: TokenIdType) -> Result<String> {
459        self.token_ids.push(token_id);
460        // log::trace!("pushed token_id: {}", token_id);
461
462        let prefix_text: String = self
463            .tokenizer
464            .decode(&self.token_ids[self.prefix_offset..self.read_offset], false)?
465            .into();
466
467        let new_result = self
468            .tokenizer
469            .decode(&self.token_ids[self.prefix_offset..], false)?;
470
471        let new_text = new_result.as_str();
472
473        // if the end character of the previous returned sequence is a multi-byte character
474        // then we can not split the text on that byte offset, so we roll back to the byte offset
475        // of the start of that character
476        let mut prefix_text_len = prefix_text.len();
477        while !new_text.is_char_boundary(prefix_text_len) && prefix_text_len > 0 {
478            prefix_text_len -= 1;
479        }
480        let prefix_text_len = prefix_text_len;
481
482        if new_text.len() > prefix_text.len() {
483            if new_result.is_partial() {
484                return Ok("".to_string());
485            } else {
486                // shift and update the state
487                let new_text = new_text[prefix_text_len..]
488                    .to_string()
489                    .replace('\u{FFFD}', "");
490                self.prefix_offset = self.read_offset;
491                self.read_offset = self.token_ids.len();
492                return Ok(new_text);
493            }
494        }
495
496        Ok("".to_string())
497    }
498
499    pub fn tokenizer(&self) -> Tokenizer {
500        self.tokenizer.clone()
501    }
502
503    pub fn token_ids(&self) -> &[TokenIdType] {
504        &self.token_ids
505    }
506
507    pub fn text(&self) -> Result<String> {
508        // let tokenizer = self.tokenizer.read().map_err(|err| {
509        //     Error::msg(format!("Failed to acquire read lock on tokenizer: {}", err))
510        // })?;
511        Ok(self.tokenizer.decode(&self.token_ids, false)?.into())
512    }
513}
514
515/// The output conditions/values of a SequenceDecoder::add_token_id operation.
516/// Result of decoding a token, indicating whether text was produced or a stop condition was met
517pub enum SequenceDecoderOutput {
518    /// The text for the appended token_id
519    Text(String),
520
521    /// A sequence of token_ids has been partially matched a stop sequence, so the text is held
522    /// until either a match or a divergence
523    Held,
524
525    /// Indicates that a stop sequence has been matched and the decoder is stopped.
526    /// Subsequent calls to append_token_id will return an error
527    Stopped,
528
529    /// Indicates that a stop token_id has been matched and the decoder is stopped.
530    /// Subsequent calls to append_token_id will return an error
531    /// The text for the stop token_id is returned
532    StoppedWithText(String),
533}
534
535/// A Sequence for decoding a stream of token ids into text and detecting stop sequences.
536/// A stop sequence is either a matching token_id or a sequence of texts/strings which match.
537/// Matches happen first at the token-level, then at the sequence-level. Hidden takes precedence
538/// over visible. For example, if you put the same token_id in both `stop_token_ids_visible` and
539/// `stop_token_ids_hidden`, the token_id will be treated as hidden.
540#[derive(Debug)]
541pub struct StopSequenceDecoder {
542    // The current sequence of token ids
543    sequence: Sequence,
544
545    // Stop Tokens - the presence of any one of these should trigger a stop
546    // If found, the text for the matched token will be returned
547    stop_token_ids_visible: Vec<TokenIdType>,
548
549    // Stop Tokens - the presence of any one of these should trigger a stop
550    // If found, the text for the matched token will NOT be returned
551    stop_token_ids_hidden: Vec<TokenIdType>,
552
553    // Stop Words - the presence of any one of these should trigger a stop
554    // If found, the text for the matched token will be returned
555    #[allow(dead_code)]
556    stop_sequences_visible: Vec<String>,
557
558    // Stop Words - the presence of any one of these should trigger a stop
559    // If found, the text for the matched token will NOT be returned
560    stop_sequences_hidden: Vec<String>,
561
562    // If the decoder has observed and returned a stop SequenceDecoderOutput,
563    // futhur calls to append_token_id will return an error
564    stopped: bool,
565
566    // text jail - if a partial stop sequence is being observed, we hold/jail the text
567    // until either the stop sequence is matched or the sequence is reset by a divergence
568    state: String,
569}
570
571impl StopSequenceDecoder {
572    /// Builder object for configurating a StopSequenceDecoder
573    pub fn builder(tokenizer: Tokenizer) -> StopSequenceDecoderBuilder {
574        StopSequenceDecoderBuilder::new(tokenizer)
575    }
576
577    /// Add a token_id to the sequence and return the SequenceDecoderOutput
578    pub fn append_token_id(&mut self, token_id: TokenIdType) -> Result<SequenceDecoderOutput> {
579        if self.stopped {
580            return Err(Error::msg("Decoder is stopped"));
581        }
582
583        // update the sequence
584        let text = self.sequence.append_token_id(token_id)?;
585
586        // append the text to the state
587        self.state.push_str(text.as_str());
588
589        let mut stop: bool = false;
590        let mut visible: bool = false;
591
592        if self.stop_token_ids_visible.contains(&token_id) {
593            stop = true;
594            visible = true;
595        }
596
597        if self.stop_token_ids_hidden.contains(&token_id) {
598            stop = true;
599            visible = false;
600        }
601
602        if stop {
603            self.stopped = true;
604            let state = std::mem::take(&mut self.state);
605            if visible {
606                return Ok(SequenceDecoderOutput::StoppedWithText(state));
607            }
608            return Ok(SequenceDecoderOutput::Stopped);
609        }
610
611        // determine if state matches any of the stop sequences
612        for stop_sequence in self.stop_sequences_hidden.iter() {
613            if stop_sequence.starts_with(&self.state) {
614                if stop_sequence == &self.state {
615                    // on matched stop sequence, we do NOT return the jailed stop sequence
616                    self.stopped = true;
617                    return Ok(SequenceDecoderOutput::Stopped);
618                } else {
619                    return Ok(SequenceDecoderOutput::Held);
620                }
621            }
622        }
623
624        let state = std::mem::take(&mut self.state);
625        Ok(SequenceDecoderOutput::Text(state))
626    }
627
628    pub fn is_empty(&self) -> bool {
629        self.sequence.token_ids.is_empty()
630    }
631
632    pub fn len(&self) -> usize {
633        self.sequence.token_ids.len()
634    }
635
636    pub fn is_complete(&self) -> bool {
637        self.stopped
638    }
639
640    pub fn close(&mut self) {
641        self.stopped = true;
642    }
643}
644
645pub struct StopSequenceDecoderBuilder {
646    tokenizer: Tokenizer,
647    stop_token_ids_visible: Vec<TokenIdType>,
648    stop_token_ids_hidden: Vec<TokenIdType>,
649    stop_sequences_visible: Vec<String>,
650    stop_sequences_hidden: Vec<String>,
651}
652
653impl StopSequenceDecoderBuilder {
654    pub fn new(tokenizer: Tokenizer) -> Self {
655        Self {
656            tokenizer,
657            stop_token_ids_visible: Vec::new(),
658            stop_token_ids_hidden: Vec::new(),
659            stop_sequences_visible: Vec::new(),
660            stop_sequences_hidden: Vec::new(),
661        }
662    }
663
664    /// Adds a visible stop token id to the StopSequenceDecoder
665    pub fn add_stop_token_id_visible(mut self, token_id: TokenIdType) -> Self {
666        self.stop_token_ids_visible.push(token_id);
667        self
668    }
669
670    /// Adds a list of visible stop token ids to the StopSequenceDecoder
671    /// Each token_id is added as for an individual match
672    pub fn add_stop_token_ids_visible(mut self, token_ids: &[TokenIdType]) -> Self {
673        self.stop_token_ids_visible.extend(token_ids);
674        self
675    }
676
677    /// Adds a hidden stop token id to the StopSequenceDecoder
678    pub fn add_stop_token_id_hidden(mut self, token_id: TokenIdType) -> Self {
679        self.stop_token_ids_hidden.push(token_id);
680        self
681    }
682
683    /// Adds a list of hidden stop token ids to the StopSequenceDecoder
684    /// Each token_id is added as for an individual match
685    pub fn add_stop_token_ids_hidden(mut self, token_ids: &[TokenIdType]) -> Self {
686        self.stop_token_ids_hidden.extend(token_ids);
687        self
688    }
689
690    pub fn add_stop_sequence_visible(mut self, text: &str) -> Self {
691        self.stop_sequences_visible.push(text.to_string());
692        self
693    }
694
695    pub fn add_stop_sequences_visible(mut self, strings: &[&str]) -> Self {
696        self.stop_sequences_visible
697            .extend(strings.iter().map(|text| text.to_string()));
698        self
699    }
700
701    pub fn add_stop_sequence_hidden(mut self, text: &str) -> Self {
702        self.stop_sequences_hidden.push(text.to_string());
703        self
704    }
705
706    pub fn add_stop_sequences_hidden(mut self, strings: &[&str]) -> Self {
707        self.stop_sequences_hidden
708            .extend(strings.iter().map(|text| text.to_string()));
709        self
710    }
711
712    pub fn build(self) -> Result<StopSequenceDecoder> {
713        Ok(StopSequenceDecoder {
714            sequence: Sequence::new(self.tokenizer.clone()),
715            stop_token_ids_visible: self.stop_token_ids_visible,
716            stop_token_ids_hidden: self.stop_token_ids_hidden,
717            stop_sequences_visible: self.stop_sequences_visible,
718            stop_sequences_hidden: self.stop_sequences_hidden,
719            stopped: false,
720            state: String::new(),
721        })
722    }
723}
dynamo_tokenizers/lib.rs

dynamo_tokenizers/
lib.rs
