dynamo_tokenizers/
lib.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4pub mod cache;
5pub mod fastokens;
6pub mod hf;
7pub mod tiktoken;
8
9// TODO: Add tokenizer benchmarks
10// TODO: Enable README.md as a module doc
11// #[doc = include_str!("../README.md")]
12
13use std::hash::{DefaultHasher, Hash, Hasher};
14use std::sync::Arc;
15use std::{fs::File, io::BufReader, ops::Deref, path::Path};
16
17use anyhow::Context as _;
18pub use anyhow::{Error, Result};
19
20pub use cache::{CacheTokenUsage, CacheTokenUsageFn, CachedTokenizer, L1CacheStats};
21pub use fastokens::FastTokenizer;
22pub use hf::HuggingFaceTokenizer;
23pub use tiktoken::TikTokenTokenizer;
24pub use traits::DecodeResult;
25
26pub type TokenIdType = u32;
27
28/// Represents the type of tokenizer being used
29#[derive(Debug)]
30pub enum TokenizerType {
31    HuggingFace(String),
32    TikToken(String),
33}
34
35/// character offsets in the original text
36pub type Offsets = (usize, usize);
37
38/// Contains the results of tokenizing text: token IDs, string tokens, and their spans
39#[derive(Debug, Clone)]
40pub enum Encoding {
41    /// Hugging Face
42    Hf(Box<tokenizers::tokenizer::Encoding>),
43    /// Sentence Piece
44    Sp(Vec<TokenIdType>),
45}
46
47impl Encoding {
48    pub fn token_ids(&self) -> &[u32] {
49        match self {
50            Encoding::Hf(inner) => inner.get_ids(),
51            Encoding::Sp(inner) => inner,
52        }
53    }
54}
55
56impl Hash for Encoding {
57    fn hash<H: Hasher>(&self, state: &mut H) {
58        self.token_ids().hash(state);
59    }
60}
61
62pub mod traits {
63    use super::*;
64
65    pub trait Encoder: Send + Sync {
66        fn encode(&self, input: &str) -> Result<Encoding>;
67        fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>>;
68    }
69
70    /// Result of decoding token IDs to text.
71    ///
72    /// Distinguishes between fully valid UTF-8 output and output that contains
73    /// trailing incomplete multi-byte sequences (represented as U+FFFD).
74    /// This lets callers like `DecodeStream::step()` decide whether to emit or
75    /// buffer without resorting to hardcoded replacement-character string checks.
76    #[derive(Debug, Clone, PartialEq, Eq, strum::EnumIs)]
77    pub enum DecodeResult {
78        /// No trailing incomplete multi-byte sequences (text does not end with U+FFFD).
79        /// Note: the string may still contain *interior* U+FFFD characters from
80        /// mid-stream invalid byte sequences; only trailing status is tracked here.
81        Complete(String),
82        /// The decoded string ends with U+FFFD, indicating incomplete trailing
83        /// multi-byte bytes that may be completed by subsequent tokens.
84        Partial(String),
85    }
86
87    impl DecodeResult {
88        /// Returns a reference to the inner string.
89        pub fn as_str(&self) -> &str {
90            match self {
91                DecodeResult::Complete(s) | DecodeResult::Partial(s) => s,
92            }
93        }
94
95        /// Construct from a decoded string: `Partial` if it ends with U+FFFD, else `Complete`.
96        pub fn from_decoded(text: String) -> Self {
97            if text.ends_with('\u{FFFD}') {
98                DecodeResult::Partial(text)
99            } else {
100                DecodeResult::Complete(text)
101            }
102        }
103    }
104
105    impl From<String> for DecodeResult {
106        fn from(text: String) -> Self {
107            DecodeResult::from_decoded(text)
108        }
109    }
110
111    impl From<DecodeResult> for String {
112        fn from(result: DecodeResult) -> Self {
113            match result {
114                DecodeResult::Complete(s) | DecodeResult::Partial(s) => s,
115            }
116        }
117    }
118
119    /// Implementations must ensure that partial multi-byte sequences produce U+FFFD
120    /// (`\u{FFFD}`) in the output rather than returning `Err`. This is commonly achieved
121    /// via `String::from_utf8_lossy` (tiktoken) or library-internal byte-fallback handling
122    /// (HuggingFace). `DecodeStream::step()` relies on `DecodeResult::Partial` to detect
123    /// incomplete sequences and buffer tokens until the full character arrives.
124    pub trait Decoder: Send + Sync {
125        fn decode(
126            &self,
127            token_ids: &[TokenIdType],
128            skip_special_tokens: bool,
129        ) -> Result<DecodeResult>;
130    }
131
132    pub trait Tokenizer: Encoder + Decoder {
133        /// Apply construction-time [`TokenizerOptions`].
134        ///
135        /// The default implementation ignores the options — correct for
136        /// tokenizers with no applicable option.
137        fn with_options(self, options: TokenizerOptions) -> Self
138        where
139            Self: Sized,
140        {
141            let _ = options;
142            self
143        }
144        // fn get_vocab_size(&self) -> usize;
145        // fn make_unique_clone(&self) -> Box<dyn Tokenizer>;
146    }
147}
148
149pub fn file_json_field<T: serde::de::DeserializeOwned>(
150    json_file_path: &Path,
151    field_name: &str,
152) -> anyhow::Result<T> {
153    let file = File::open(json_file_path)
154        .with_context(|| format!("Failed to open file: {:?}", json_file_path))?;
155    let reader = BufReader::new(file);
156
157    let json_data: serde_json::Value = serde_json::from_reader(reader)
158        .with_context(|| format!("Failed to parse JSON from file: {:?}", json_file_path))?;
159
160    let map = json_data.as_object().ok_or_else(|| {
161        anyhow::anyhow!("JSON root is not an object in file: {:?}", json_file_path)
162    })?;
163
164    let field_value = map.get(field_name).ok_or_else(|| {
165        anyhow::anyhow!(
166            "Field '{}' not found in JSON file: {:?}",
167            field_name,
168            json_file_path
169        )
170    })?;
171
172    serde_json::from_value(field_value.clone()).with_context(|| {
173        format!(
174            "Failed to deserialize field '{}' (value: {:?}) to the expected type from file: {:?}",
175            field_name, field_value, json_file_path
176        )
177    })
178}
179
180pub fn log_json_err(filename: &str, json: &str, err: &serde_json::Error) {
181    const ERROR_PREFIX: &str = ">>     ";
182
183    if !(err.is_syntax() || err.is_data()) {
184        return;
185    }
186
187    let line = err.line().saturating_sub(1);
188    let column = err.column().saturating_sub(1);
189
190    let json_lines: Vec<&str> = json.lines().collect();
191    if json_lines.is_empty() {
192        tracing::error!("JSON parsing error in {filename}: File is empty.");
193        return;
194    }
195
196    let start_index = line.saturating_sub(2);
197    let end_index = line.saturating_add(3).min(json_lines.len());
198
199    let mut context_lines: Vec<String> = (start_index..end_index)
200        .map(|i| {
201            if i == line {
202                format!("{ERROR_PREFIX}{}", json_lines[i])
203            } else {
204                format!("{:06} {}", i + 1, json_lines[i])
205            }
206        })
207        .collect();
208
209    let col_indicator = "_".to_string().repeat(column + ERROR_PREFIX.len()) + "^";
210    let error_in_context_idx = line - start_index;
211    if error_in_context_idx < context_lines.len() {
212        context_lines.insert(error_in_context_idx + 1, col_indicator);
213    }
214
215    tracing::error!(
216        "JSON parsing error in {filename}: Line {}, column {}:\n{}",
217        err.line(),
218        err.column(),
219        context_lines.join("\n")
220    );
221}
222
223impl Encoding {
224    pub fn get_hash(&self) -> u64 {
225        let mut hasher = DefaultHasher::new();
226        self.hash(&mut hasher);
227        hasher.finish()
228    }
229}
230
231/// Construction options for [`Tokenizer::from_file_with_options`] /
232/// [`create_tokenizer_from_file`], applied to concrete tokenizers via
233/// [`traits::Tokenizer::with_options`].
234#[derive(Debug, Clone, Copy, Default)]
235pub struct TokenizerOptions {
236    /// Ask the tokenizer to add its declared special tokens (e.g. BOS/EOS via
237    /// the HuggingFace post-processor) during `encode`/`encode_batch`.
238    /// Defaults to `false`, the historical behavior.
239    ///
240    /// Only applicable to HuggingFace tokenizers.
241    pub add_special_tokens: bool,
242}
243
244/// Main tokenizer wrapper that provides a unified interface for different tokenizer implementations
245#[derive(Clone)]
246pub struct Tokenizer(Arc<dyn traits::Tokenizer>);
247
248impl Tokenizer {
249    pub fn from_file(file_path: &str) -> Result<Tokenizer> {
250        Ok(Tokenizer(create_tokenizer_from_file(file_path)?))
251    }
252
253    pub fn from_file_with_options(file_path: &str, options: TokenizerOptions) -> Result<Tokenizer> {
254        Ok(Tokenizer(create_tokenizer_from_file_with_options(
255            file_path, options,
256        )?))
257    }
258
259    /// Create a stateful sequence object for decoding token_ids into text
260    pub fn decode_stream(
261        &self,
262        prompt_token_ids: &[TokenIdType],
263        skip_special_tokens: bool,
264    ) -> DecodeStream {
265        DecodeStream::new(self.0.clone(), prompt_token_ids, skip_special_tokens)
266    }
267}
268
269impl Deref for Tokenizer {
270    type Target = Arc<dyn traits::Tokenizer>;
271
272    fn deref(&self) -> &Self::Target {
273        &self.0
274    }
275}
276
277impl From<Arc<dyn traits::Tokenizer>> for Tokenizer {
278    fn from(tokenizer: Arc<dyn traits::Tokenizer>) -> Self {
279        Tokenizer(tokenizer)
280    }
281}
282
283impl<T> From<Arc<T>> for Tokenizer
284where
285    T: traits::Tokenizer + 'static, // 'static is required to ensure T can be safely put into an Arc
286{
287    fn from(tokenizer: Arc<T>) -> Self {
288        Tokenizer(tokenizer)
289    }
290}
291
292/// Create a tokenizer from a file path to a tokenizer file.
293/// The file extension is used to determine the tokenizer type.
294/// Supported file types are:
295/// - json: HuggingFace tokenizer
296/// - model, tiktoken: tiktoken BPE tokenizer (requires `config.json` with a supported
297///   `model_type` in the same directory; currently: kimi, kimi_k2, kimi_k25)
298pub fn create_tokenizer_from_file(file_path: &str) -> Result<Arc<dyn traits::Tokenizer>> {
299    create_tokenizer_from_file_with_options(file_path, Default::default())
300}
301
302/// Create a tokenizer from a file path to a tokenizer file with additional tokenizer option.
303/// The file extension is used to determine the tokenizer type.
304/// Supported file types are:
305/// - json: HuggingFace tokenizer
306/// - model, tiktoken: tiktoken BPE tokenizer (requires `config.json` with a supported
307///   `model_type` in the same directory; currently: kimi, kimi_k2, kimi_k25)
308pub fn create_tokenizer_from_file_with_options(
309    file_path: &str,
310    options: TokenizerOptions,
311) -> Result<Arc<dyn traits::Tokenizer>> {
312    use traits::Tokenizer as _;
313
314    let path = Path::new(file_path);
315    let extension = path
316        .extension()
317        .and_then(std::ffi::OsStr::to_str)
318        .ok_or_else(|| Error::msg("Failed to read file extension".to_string()))?;
319
320    match extension {
321        "json" => {
322            let tokenizer = HuggingFaceTokenizer::from_file(file_path)?.with_options(options);
323            Ok(Arc::new(tokenizer))
324        }
325        "model" | "tiktoken" => {
326            let tokenizer = TikTokenTokenizer::from_file_auto(file_path)?.with_options(options);
327            Ok(Arc::new(tokenizer))
328        }
329        _ => Err(Error::msg(format!(
330            "Unsupported tokenizer file type: .{extension}"
331        ))),
332    }
333}
334
335// With incremental detokenization, we need to consider the final context tokens when handling the initial decode tokens.
336// This is the initial offset from the end of the context that we start decoding from.
337// Both Huggingface TGI and vLLM use this same value.
338// See: https://github.com/huggingface/text-generation-inference/blob/24c2bff65924801ddf90fa24fcc72752d4f45538/server/text_generation_server/models/mamba.py#L169
339// and https://github.com/vllm-project/vllm/blob/da2705198fa19030a25d0bea437f7be6547d47d4/vllm/transformers_utils/detokenizer_utils.py#L51
340const INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET: usize = 5;
341
342/// DecodeStream will keep the state necessary to produce individual chunks of
343/// strings given an input stream of token_ids.
344///
345/// This is necessary because decoding in general cannot achieve that since strings
346/// depend on surrounding ids to provide a valid string. Typically stripping extra spaces.
347pub struct DecodeStream {
348    /// The tokenizer used to decode token_ids
349    tokenizer: Arc<dyn traits::Tokenizer>,
350
351    skip_special_tokens: bool,
352    /// A temporary buffer of the necessary token_ids needed
353    /// to produce valid string chunks.
354    /// This typically contains 3 parts:
355    ///  - read
356    ///  - prefix
357    ///  - rest
358    ///
359    /// Read is the bit necessary to surround the prefix
360    /// so decoding the whole ids produces a valid prefix.
361    /// Prefix is the previously produced string, kept around to trim off of
362    /// the next valid chunk
363    all_token_ids: Vec<u32>,
364
365    prefix_offset: usize,
366
367    read_offset: usize,
368}
369
370impl DecodeStream {
371    pub fn new(
372        tokenizer: Arc<dyn traits::Tokenizer>,
373        prompt_token_ids: &[TokenIdType],
374        skip_special_tokens: bool,
375    ) -> Self {
376        let num_input_tokens = prompt_token_ids.len();
377        let prompt_token_ids = prompt_token_ids.to_vec();
378        Self {
379            tokenizer,
380            skip_special_tokens,
381            all_token_ids: prompt_token_ids,
382            prefix_offset: num_input_tokens
383                .saturating_sub(INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET),
384            read_offset: num_input_tokens,
385        }
386    }
387
388    /// Step appends a token_id to the internal state and tries to produce a text chunk.
389    ///
390    /// Implementation directly copied from Huggingface's TGI:
391    /// https://github.com/huggingface/text-generation-inference/blob/24c2bff65924801ddf90fa24fcc72752d4f45538/server/text_generation_server/models/model.py#L144
392    ///
393    /// Returning `None` means the given id is not enough to produce a chunk.
394    /// This typically happens with `byte_fallback` options where some tokens do not
395    /// represent valid UTF-8, and only follow-up token_ids will help produce
396    /// a valid chunk.
397    pub fn step(&mut self, id: u32) -> Result<Option<String>> {
398        self.all_token_ids.push(id);
399
400        let prefix_text: String = self
401            .tokenizer
402            .decode(
403                &self.all_token_ids[self.prefix_offset..self.read_offset],
404                self.skip_special_tokens,
405            )?
406            .into();
407
408        let new_result = self.tokenizer.decode(
409            &self.all_token_ids[self.prefix_offset..],
410            self.skip_special_tokens,
411        )?;
412
413        let new_text = new_result.as_str();
414        if new_text.len() > prefix_text.len() && !new_result.is_partial() {
415            let emitted = new_text[prefix_text.len()..].to_string();
416
417            self.prefix_offset = self.read_offset;
418            self.read_offset = self.all_token_ids.len();
419
420            Ok(Some(emitted))
421        } else {
422            Ok(None)
423        }
424    }
425}
426
427/// Maintains state for an ongoing sequence of tokens and their decoded text
428pub struct Sequence {
429    /// Encodes text -> token_ids
430    tokenizer: Tokenizer,
431
432    /// The current sequence of token ids
433    token_ids: Vec<TokenIdType>,
434
435    /// The position in the current sequence the last decoded token completed
436    prefix_offset: usize,
437
438    /// Current position in the sequence
439    read_offset: usize,
440}
441
442impl std::fmt::Debug for Sequence {
443    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
444        f.debug_struct("Sequence")
445            .field("tokenizer", &"Arc<dyn Tokenizer>")
446            .field(
447                "token_ids",
448                &format_args!("{}", {
449                    let token_ids = self.token_ids();
450                    if token_ids.len() <= 20 {
451                        format!("{:?}", token_ids)
452                    } else {
453                        let first_ten = &token_ids[..10];
454                        let last_ten = &token_ids[token_ids.len() - 10..];
455                        format!("{:?} ... {:?}", first_ten, last_ten)
456                    }
457                }),
458            )
459            .field("prefix_offset", &self.prefix_offset)
460            .field("read_offset", &self.read_offset)
461            .field("token count", &self.token_ids.len())
462            .finish()
463    }
464}
465
466impl Sequence {
467    pub fn new(tokenizer: Tokenizer) -> Self {
468        Self {
469            tokenizer,
470            token_ids: Vec::new(),
471            prefix_offset: 0,
472            read_offset: 0,
473        }
474    }
475
476    pub fn is_empty(&self) -> bool {
477        self.token_ids.is_empty()
478    }
479
480    pub fn len(&self) -> usize {
481        self.token_ids.len()
482    }
483
484    pub fn clear(&mut self) {
485        self.token_ids.clear();
486        self.prefix_offset = 0;
487        self.read_offset = 0;
488    }
489
490    pub fn append_text(&mut self, input: &str) -> Result<()> {
491        // let tokenizer = self.tokenizer.read().map_err(|err| {
492        //     Error::msg(format!("Failed to acquire read lock on tokenizer: {}", err))
493        // })?;
494
495        let encoding = self.tokenizer.encode(input)?;
496        self.token_ids.extend(encoding.token_ids());
497        Ok(())
498    }
499
500    // Based on
501    // https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
502    // under Apache 2.0 license
503    pub fn append_token_id(&mut self, token_id: TokenIdType) -> Result<String> {
504        self.token_ids.push(token_id);
505        // log::trace!("pushed token_id: {}", token_id);
506
507        let prefix_text: String = self
508            .tokenizer
509            .decode(&self.token_ids[self.prefix_offset..self.read_offset], false)?
510            .into();
511
512        let new_result = self
513            .tokenizer
514            .decode(&self.token_ids[self.prefix_offset..], false)?;
515
516        let new_text = new_result.as_str();
517
518        // if the end character of the previous returned sequence is a multi-byte character
519        // then we can not split the text on that byte offset, so we roll back to the byte offset
520        // of the start of that character
521        let mut prefix_text_len = prefix_text.len();
522        while !new_text.is_char_boundary(prefix_text_len) && prefix_text_len > 0 {
523            prefix_text_len -= 1;
524        }
525        let prefix_text_len = prefix_text_len;
526
527        if new_text.len() > prefix_text.len() {
528            if new_result.is_partial() {
529                return Ok("".to_string());
530            } else {
531                // shift and update the state
532                let new_text = new_text[prefix_text_len..]
533                    .to_string()
534                    .replace('\u{FFFD}', "");
535                self.prefix_offset = self.read_offset;
536                self.read_offset = self.token_ids.len();
537                return Ok(new_text);
538            }
539        }
540
541        Ok("".to_string())
542    }
543
544    pub fn tokenizer(&self) -> Tokenizer {
545        self.tokenizer.clone()
546    }
547
548    pub fn token_ids(&self) -> &[TokenIdType] {
549        &self.token_ids
550    }
551
552    pub fn text(&self) -> Result<String> {
553        // let tokenizer = self.tokenizer.read().map_err(|err| {
554        //     Error::msg(format!("Failed to acquire read lock on tokenizer: {}", err))
555        // })?;
556        Ok(self.tokenizer.decode(&self.token_ids, false)?.into())
557    }
558}
559
560/// The output conditions/values of a SequenceDecoder::add_token_id operation.
561/// Result of decoding a token, indicating whether text was produced or a stop condition was met
562pub enum SequenceDecoderOutput {
563    /// The text for the appended token_id
564    Text(String),
565
566    /// A sequence of token_ids has been partially matched a stop sequence, so the text is held
567    /// until either a match or a divergence
568    Held,
569
570    /// Indicates that a stop sequence has been matched and the decoder is stopped.
571    /// Subsequent calls to append_token_id will return an error
572    Stopped,
573
574    /// Indicates that a stop token_id has been matched and the decoder is stopped.
575    /// Subsequent calls to append_token_id will return an error
576    /// The text for the stop token_id is returned
577    StoppedWithText(String),
578}
579
580/// A Sequence for decoding a stream of token ids into text and detecting stop sequences.
581/// A stop sequence is either a matching token_id or a sequence of texts/strings which match.
582/// Matches happen first at the token-level, then at the sequence-level. Hidden takes precedence
583/// over visible. For example, if you put the same token_id in both `stop_token_ids_visible` and
584/// `stop_token_ids_hidden`, the token_id will be treated as hidden.
585#[derive(Debug)]
586pub struct StopSequenceDecoder {
587    // The current sequence of token ids
588    sequence: Sequence,
589
590    // Stop Tokens - the presence of any one of these should trigger a stop
591    // If found, the text for the matched token will be returned
592    stop_token_ids_visible: Vec<TokenIdType>,
593
594    // Stop Tokens - the presence of any one of these should trigger a stop
595    // If found, the text for the matched token will NOT be returned
596    stop_token_ids_hidden: Vec<TokenIdType>,
597
598    // Stop Words - the presence of any one of these should trigger a stop
599    // If found, the text for the matched token will be returned
600    #[allow(dead_code)]
601    stop_sequences_visible: Vec<String>,
602
603    // Stop Words - the presence of any one of these should trigger a stop
604    // If found, the text for the matched token will NOT be returned
605    stop_sequences_hidden: Vec<String>,
606
607    // If the decoder has observed and returned a stop SequenceDecoderOutput,
608    // futhur calls to append_token_id will return an error
609    stopped: bool,
610
611    // text jail - if a partial stop sequence is being observed, we hold/jail the text
612    // until either the stop sequence is matched or the sequence is reset by a divergence
613    state: String,
614}
615
616impl StopSequenceDecoder {
617    /// Builder object for configurating a StopSequenceDecoder
618    pub fn builder(tokenizer: Tokenizer) -> StopSequenceDecoderBuilder {
619        StopSequenceDecoderBuilder::new(tokenizer)
620    }
621
622    /// Add a token_id to the sequence and return the SequenceDecoderOutput
623    pub fn append_token_id(&mut self, token_id: TokenIdType) -> Result<SequenceDecoderOutput> {
624        if self.stopped {
625            return Err(Error::msg("Decoder is stopped"));
626        }
627
628        // update the sequence
629        let text = self.sequence.append_token_id(token_id)?;
630
631        // append the text to the state
632        self.state.push_str(text.as_str());
633
634        let mut stop: bool = false;
635        let mut visible: bool = false;
636
637        if self.stop_token_ids_visible.contains(&token_id) {
638            stop = true;
639            visible = true;
640        }
641
642        if self.stop_token_ids_hidden.contains(&token_id) {
643            stop = true;
644            visible = false;
645        }
646
647        if stop {
648            self.stopped = true;
649            let state = std::mem::take(&mut self.state);
650            if visible {
651                return Ok(SequenceDecoderOutput::StoppedWithText(state));
652            }
653            return Ok(SequenceDecoderOutput::Stopped);
654        }
655
656        // determine if state matches any of the stop sequences
657        for stop_sequence in self.stop_sequences_hidden.iter() {
658            if stop_sequence.starts_with(&self.state) {
659                if stop_sequence == &self.state {
660                    // on matched stop sequence, we do NOT return the jailed stop sequence
661                    self.stopped = true;
662                    return Ok(SequenceDecoderOutput::Stopped);
663                } else {
664                    return Ok(SequenceDecoderOutput::Held);
665                }
666            }
667        }
668
669        let state = std::mem::take(&mut self.state);
670        Ok(SequenceDecoderOutput::Text(state))
671    }
672
673    pub fn is_empty(&self) -> bool {
674        self.sequence.token_ids.is_empty()
675    }
676
677    pub fn len(&self) -> usize {
678        self.sequence.token_ids.len()
679    }
680
681    pub fn is_complete(&self) -> bool {
682        self.stopped
683    }
684
685    pub fn close(&mut self) {
686        self.stopped = true;
687    }
688}
689
690pub struct StopSequenceDecoderBuilder {
691    tokenizer: Tokenizer,
692    stop_token_ids_visible: Vec<TokenIdType>,
693    stop_token_ids_hidden: Vec<TokenIdType>,
694    stop_sequences_visible: Vec<String>,
695    stop_sequences_hidden: Vec<String>,
696}
697
698impl StopSequenceDecoderBuilder {
699    pub fn new(tokenizer: Tokenizer) -> Self {
700        Self {
701            tokenizer,
702            stop_token_ids_visible: Vec::new(),
703            stop_token_ids_hidden: Vec::new(),
704            stop_sequences_visible: Vec::new(),
705            stop_sequences_hidden: Vec::new(),
706        }
707    }
708
709    /// Adds a visible stop token id to the StopSequenceDecoder
710    pub fn add_stop_token_id_visible(mut self, token_id: TokenIdType) -> Self {
711        self.stop_token_ids_visible.push(token_id);
712        self
713    }
714
715    /// Adds a list of visible stop token ids to the StopSequenceDecoder
716    /// Each token_id is added as for an individual match
717    pub fn add_stop_token_ids_visible(mut self, token_ids: &[TokenIdType]) -> Self {
718        self.stop_token_ids_visible.extend(token_ids);
719        self
720    }
721
722    /// Adds a hidden stop token id to the StopSequenceDecoder
723    pub fn add_stop_token_id_hidden(mut self, token_id: TokenIdType) -> Self {
724        self.stop_token_ids_hidden.push(token_id);
725        self
726    }
727
728    /// Adds a list of hidden stop token ids to the StopSequenceDecoder
729    /// Each token_id is added as for an individual match
730    pub fn add_stop_token_ids_hidden(mut self, token_ids: &[TokenIdType]) -> Self {
731        self.stop_token_ids_hidden.extend(token_ids);
732        self
733    }
734
735    pub fn add_stop_sequence_visible(mut self, text: &str) -> Self {
736        self.stop_sequences_visible.push(text.to_string());
737        self
738    }
739
740    pub fn add_stop_sequences_visible(mut self, strings: &[&str]) -> Self {
741        self.stop_sequences_visible
742            .extend(strings.iter().map(|text| text.to_string()));
743        self
744    }
745
746    pub fn add_stop_sequence_hidden(mut self, text: &str) -> Self {
747        self.stop_sequences_hidden.push(text.to_string());
748        self
749    }
750
751    pub fn add_stop_sequences_hidden(mut self, strings: &[&str]) -> Self {
752        self.stop_sequences_hidden
753            .extend(strings.iter().map(|text| text.to_string()));
754        self
755    }
756
757    pub fn build(self) -> Result<StopSequenceDecoder> {
758        Ok(StopSequenceDecoder {
759            sequence: Sequence::new(self.tokenizer.clone()),
760            stop_token_ids_visible: self.stop_token_ids_visible,
761            stop_token_ids_hidden: self.stop_token_ids_hidden,
762            stop_sequences_visible: self.stop_sequences_visible,
763            stop_sequences_hidden: self.stop_sequences_hidden,
764            stopped: false,
765            state: String::new(),
766        })
767    }
768}
dynamo_tokenizers/lib.rs

dynamo_tokenizers/
lib.rs
