Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Wrapper around tokenizers::Tokenizer and minijinja::Environment providing more utilities.

Implementations

impl Tokenizer

pub fn from_tokenizer(tokenizer: Tokenizer) -> Self

Wraps a Hugging Face tokenizer with chat-template support.

pub fn set_template_kwargs(&mut self, template_kwargs: Map<String, Value>)

Replaces the default variables supplied to chat templates.

pub fn template_kwargs(&self) -> &Map<String, Value>

Returns the default variables supplied to chat templates.

pub fn from_file(file: impl AsRef<Path>) -> Result<Self>

Loads and wraps a tokenizer from a tokenizer.json file.

pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self>

Loads and wraps a tokenizer from serialized tokenizer.json bytes.

pub fn apply_chat_template<'a, I, R, T>( &'a mut self, model_template: impl Into<ModelChatTemplate>, args: ApplyChatTemplateArgs<'a, I, R, T>, ) -> Result<Vec<String>, Error>

where I: IntoIterator<Item = Chat<'a, R, T>>, R: Serialize + 'a, T: Serialize + 'a,

Renders a batch of structured conversations with a model chat template.

pub fn apply_chat_template_and_encode<'a, I, R, T>( &mut self, model_template: impl Into<ModelChatTemplate>, args: ApplyChatTemplateArgs<'a, I, R, T>, ) -> Result<Vec<Encoding>, Error>

where I: IntoIterator<Item = Chat<'a, R, T>>, R: Serialize + 'a, T: Serialize + 'a,

Renders and tokenizes a batch of structured conversations.

pub fn apply_chat_template_json<'a, I>( &mut self, model_template: impl Into<ModelChatTemplate>, conversations: I, tools: Option<&'a [Value]>, model_id: &'a str, add_generation_prompt: bool, template_kwargs: Option<&'a Map<String, Value>>, ) -> Result<Vec<String>, Error>

where I: IntoIterator<Item = Vec<Value>>,

Renders conversations already represented as JSON message arrays.

Methods from Deref<Target = TokenizerImpl<ModelWrapper, NormalizerWrapper, PreTokenizerWrapper, PostProcessorWrapper, DecoderWrapper>>

pub fn get_normalizer(&self) -> Option<&N>

Get the normalizer

pub fn get_pre_tokenizer(&self) -> Option<&PT>

Get the pre tokenizer

pub fn get_post_processor(&self) -> Option<&PP>

Get the post processor

pub fn get_decoder(&self) -> Option<&D>

Get the decoder

pub fn get_model(&self) -> &M

Get the model

pub fn get_added_vocabulary(&self) -> &AddedVocabulary

Get the added vocabulary

pub fn get_truncation(&self) -> Option<&TruncationParams>

Get the currently set truncation parameters

pub fn get_padding(&self) -> Option<&PaddingParams>

Get the currently set padding parameters

pub fn get_vocab(&self, with_added_tokens: bool) -> HashMap<String, u32>

pub fn get_added_tokens_decoder(&self) -> AHashMap<u32, AddedToken>

Get the added tokens decoder

pub fn get_vocab_size(&self, with_added_tokens: bool) -> usize

Get the size of the vocabulary

pub fn token_to_id(&self, token: &str) -> Option<u32>

Converts a token in the corresponding id.

pub fn id_to_token(&self, id: u32) -> Option<String>

Converts an id to the corresponding token.

pub fn get_encode_special_tokens(&self) -> bool

Get added token value

pub fn encode_fast<'s, E>( &self, input: E, add_special_tokens: bool, ) -> Result<Encoding, Box<dyn Error + Send + Sync>>

where E: Into<EncodeInput<'s>>,

Encode the given input. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly: Contrarily to encode, it does not compute offsets

// Sequences:
tokenizer.encode_fast("Single sequence", false);
tokenizer.encode_fast(("Sequence A", "Sequence B"), false);

// Pre-tokenized sequences:
tokenizer.encode_fast(&["Single", "sequence"][..], false);
tokenizer.encode_fast((
    &["Sequence", "A"][..],
    &["Sequence", "B"][..]
), false);

// or even both types together:
tokenizer.encode_fast(("A complete sequence", &["And", "a", "tokenized"][..]), false);

pub fn encode<'s, E>( &self, input: E, add_special_tokens: bool, ) -> Result<Encoding, Box<dyn Error + Send + Sync>>

where E: Into<EncodeInput<'s>>,

Encode the given input. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly:

// Sequences:
tokenizer.encode("Single sequence", false);
tokenizer.encode(("Sequence A", "Sequence B"), false);

// Pre-tokenized sequences:
tokenizer.encode(&["Single", "sequence"][..], false);
tokenizer.encode((
    &["Sequence", "A"][..],
    &["Sequence", "B"][..]
), false);

// or even both types together:
tokenizer.encode(("A complete sequence", &["And", "a", "tokenized"][..]), false);

pub fn encode_char_offsets<'s, E>( &self, input: E, add_special_tokens: bool, ) -> Result<Encoding, Box<dyn Error + Send + Sync>>

where E: Into<EncodeInput<'s>>,

Encode the given input, using offsets relative to chars instead of bytes. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly:

// Sequences:
tokenizer.encode("Single sequence", false);
tokenizer.encode(("Sequence A", "Sequence B"), false);

// Pre-tokenized sequences:
tokenizer.encode(&["Single", "sequence"][..], false);
tokenizer.encode((
    &["Sequence", "A"][..],
    &["Sequence", "B"][..]
), false);

// or even both types together:
tokenizer.encode(("A complete sequence", &["And", "a", "tokenized"][..]), false);

pub fn decode( &self, ids: &[u32], skip_special_tokens: bool, ) -> Result<String, Box<dyn Error + Send + Sync>>

Decode the given ids, back to a String

pub fn decode_stream( &self, skip_special_tokens: bool, ) -> DecodeStream<'_, M, N, PT, PP, D>

Decode the given ids, back to a String See DecodeStream

pub fn post_process( &self, encoding: Encoding, pair_encoding: Option<Encoding>, add_special_tokens: bool, ) -> Result<Encoding, Box<dyn Error + Send + Sync>>

Post processing logic, handling the case where there is no PostProcessor set

pub fn encode_batch<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool, ) -> Result<Vec<Encoding>, Box<dyn Error + Send + Sync>>

where E: Into<EncodeInput<'s>> + Send,

Encode all the sentences in parallel, using multiple threads

pub fn encode_batch_char_offsets<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool, ) -> Result<Vec<Encoding>, Box<dyn Error + Send + Sync>>

where E: Into<EncodeInput<'s>> + Send,

Encode all the sentences in parallel, using multiple threads. The offsets on each Encoding will be relative to chars instead of bytes.

pub fn encode_batch_fast<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool, ) -> Result<Vec<Encoding>, Box<dyn Error + Send + Sync>>

where E: Into<EncodeInput<'s>> + Send,

Encode all the sentences in parallel, using multiple threads

pub fn decode_batch( &self, sentences: &[&[u32]], skip_special_tokens: bool, ) -> Result<Vec<String>, Box<dyn Error + Send + Sync>>

where M: Send + Sync,

Decode all sentences in parallel

pub fn to_string( &self, pretty: bool, ) -> Result<String, Box<dyn Error + Send + Sync>>

Serialize the current tokenizer as a String

pub fn save<P>( &self, path: P, pretty: bool, ) -> Result<(), Box<dyn Error + Send + Sync>>

where P: AsRef<Path>,

Save the current tokenizer at the given path

Trait Implementations

impl Deref for Tokenizer

type Target = Tokenizer

The resulting type after dereferencing.

fn deref(&self) -> &Self::Target

Dereferences the value.

impl DerefMut for Tokenizer

fn deref_mut(&mut self) -> &mut Self::Target

Mutably dereferences the value.

impl FromStr for Tokenizer

type Err = Box<dyn Error + Send + Sync>

The associated error which can be returned from parsing.

fn from_str(s: &str) -> Result<Self, Self::Err>

Parses a string s to return a value of this type.