Struct TokenizerImpl

pub struct TokenizerImpl<M, N, PT, PP, D> { /* private fields */ }

A Tokenizer is capable of encoding/decoding any text.

Implementations

impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D>

where M: Model, N: Normalizer, PT: PreTokenizer, PP: PostProcessor, D: Decoder,

pub fn new(model: M) -> Self

Instantiate a new Tokenizer, with the given Model

pub fn with_normalizer(&mut self, normalizer: Option<impl Into<N>>) -> &mut Self

Set the normalizer

pub fn get_normalizer(&self) -> Option<&N>

Get the normalizer

pub fn with_pre_tokenizer( &mut self, pre_tokenizer: Option<impl Into<PT>>, ) -> &mut Self

Set the pre tokenizer

pub fn get_pre_tokenizer(&self) -> Option<&PT>

Get the pre tokenizer

pub fn with_post_processor( &mut self, post_processor: Option<impl Into<PP>>, ) -> &mut Self

Set the post processor

pub fn get_post_processor(&self) -> Option<&PP>

Get the post processor

pub fn with_decoder(&mut self, decoder: Option<impl Into<D>>) -> &mut Self

Set the decoder

pub fn get_decoder(&self) -> Option<&D>

Get the decoder

pub fn with_model(&mut self, model: impl Into<M>) -> &mut Self

Set the model

pub fn get_model(&self) -> &M

Get the model

pub fn with_added_vocabulary( &mut self, added_vocabulary: AddedVocabulary, ) -> &mut Self

Set the added vocabulary.

pub fn get_added_vocabulary(&self) -> &AddedVocabulary

Get the added vocabulary

pub fn with_truncation( &mut self, trunc: Option<TruncationParams>, ) -> Result<&mut Self>

Set the truncation parameters

Fails if stride is too high relative to max_length and post_processor.added_tokens()

pub fn get_truncation(&self) -> Option<&TruncationParams>

Get the currently set truncation parameters

pub fn get_truncation_mut(&mut self) -> Option<&mut TruncationParams>

Get a mutable reference to the currently set truncation parameters

pub fn with_padding(&mut self, padding: Option<PaddingParams>) -> &mut Self

Set the padding parameters

pub fn get_padding(&self) -> Option<&PaddingParams>

Get the currently set padding parameters

pub fn get_padding_mut(&mut self) -> Option<&mut PaddingParams>

Get a mutable reference to the currently set padding parameters

pub fn get_vocab(&self, with_added_tokens: bool) -> HashMap<String, u32>

pub fn get_added_tokens_decoder(&self) -> AHashMap<u32, AddedToken>

Get the added tokens decoder

pub fn get_vocab_size(&self, with_added_tokens: bool) -> usize

Get the size of the vocabulary

pub fn token_to_id(&self, token: &str) -> Option<u32>

Converts a token in the corresponding id.

pub fn id_to_token(&self, id: u32) -> Option<String>

Converts an id to the corresponding token.

pub fn set_encode_special_tokens(&mut self, value: bool)

set the added vocab’s splitting scheme

pub fn get_encode_special_tokens(&self) -> bool

Get added token value

pub fn encode_fast<'s, E>( &self, input: E, add_special_tokens: bool, ) -> Result<Encoding>

where E: Into<EncodeInput<'s>>,

Encode the given input. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly: Contrarily to encode, it does not compute offsets

// Sequences:
tokenizer.encode_fast("Single sequence", false);
tokenizer.encode_fast(("Sequence A", "Sequence B"), false);

// Pre-tokenized sequences:
tokenizer.encode_fast(&["Single", "sequence"][..], false);
tokenizer.encode_fast((
    &["Sequence", "A"][..],
    &["Sequence", "B"][..]
), false);

// or even both types together:
tokenizer.encode_fast(("A complete sequence", &["And", "a", "tokenized"][..]), false);

pub fn encode<'s, E>( &self, input: E, add_special_tokens: bool, ) -> Result<Encoding>

where E: Into<EncodeInput<'s>>,

Encode the given input. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly:

// Sequences:
tokenizer.encode("Single sequence", false);
tokenizer.encode(("Sequence A", "Sequence B"), false);

// Pre-tokenized sequences:
tokenizer.encode(&["Single", "sequence"][..], false);
tokenizer.encode((
    &["Sequence", "A"][..],
    &["Sequence", "B"][..]
), false);

// or even both types together:
tokenizer.encode(("A complete sequence", &["And", "a", "tokenized"][..]), false);

pub fn encode_char_offsets<'s, E>( &self, input: E, add_special_tokens: bool, ) -> Result<Encoding>

where E: Into<EncodeInput<'s>>,

Encode the given input, using offsets relative to chars instead of bytes. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly:

// Sequences:
tokenizer.encode("Single sequence", false);
tokenizer.encode(("Sequence A", "Sequence B"), false);

// Pre-tokenized sequences:
tokenizer.encode(&["Single", "sequence"][..], false);
tokenizer.encode((
    &["Sequence", "A"][..],
    &["Sequence", "B"][..]
), false);

// or even both types together:
tokenizer.encode(("A complete sequence", &["And", "a", "tokenized"][..]), false);

pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String>

Decode the given ids, back to a String

pub fn decode_stream( &self, skip_special_tokens: bool, ) -> DecodeStream<'_, M, N, PT, PP, D>

Decode the given ids, back to a String See DecodeStream

impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D>

where N: Normalizer, M: Model,

pub fn add_special_tokens(&mut self, tokens: &[AddedToken]) -> usize

Register the given tokens as special tokens. This is especially useful for removing these special tokens while decoding

pub fn add_tokens(&mut self, tokens: &[AddedToken]) -> usize

Add the given tokens to the added vocabulary

impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D>

where PP: PostProcessor,

pub fn post_process( &self, encoding: Encoding, pair_encoding: Option<Encoding>, add_special_tokens: bool, ) -> Result<Encoding>

Post processing logic, handling the case where there is no PostProcessor set

impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D>

where M: Model + Send + Sync, N: Normalizer + Send + Sync, PT: PreTokenizer + Send + Sync, PP: PostProcessor + Send + Sync, D: Decoder + Send + Sync,

pub fn encode_batch<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool, ) -> Result<Vec<Encoding>>

where E: Into<EncodeInput<'s>> + Send,

Encode all the sentences in parallel, using multiple threads

pub fn encode_batch_char_offsets<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool, ) -> Result<Vec<Encoding>>

where E: Into<EncodeInput<'s>> + Send,

Encode all the sentences in parallel, using multiple threads. The offsets on each Encoding will be relative to chars instead of bytes.

pub fn encode_batch_fast<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool, ) -> Result<Vec<Encoding>>

where E: Into<EncodeInput<'s>> + Send,

Encode all the sentences in parallel, using multiple threads

pub fn decode_batch( &self, sentences: &[&[u32]], skip_special_tokens: bool, ) -> Result<Vec<String>>

where M: Send + Sync,

Decode all sentences in parallel

pub fn train_from_files<T>( &mut self, trainer: &mut T, files: Vec<String>, ) -> Result<&mut Self>

where T: Trainer<Model = M> + Sync,

Train our Model from files

pub fn train<T, I, S>( &mut self, trainer: &mut T, sequences: I, ) -> Result<&mut Self>

where T: Trainer<Model = M> + Sync, I: Iterator<Item = S> + Send, S: AsRef<str> + Send,

Train our Model, using the given Trainer and iterator

impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D>

where M: DeserializeOwned + Model, N: DeserializeOwned + Normalizer, PT: DeserializeOwned + PreTokenizer, PP: DeserializeOwned + PostProcessor, D: DeserializeOwned + Decoder,

pub fn from_file<P: AsRef<Path>>(file: P) -> Result<Self>

Instantiate a new Tokenizer from the given file

impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D>

where M: DeserializeOwned + Model, N: DeserializeOwned + Normalizer, PT: DeserializeOwned + PreTokenizer, PP: DeserializeOwned + PostProcessor, D: DeserializeOwned + Decoder,

pub fn from_bytes<P: AsRef<[u8]>>(bytes: P) -> Result<Self>

Instantiate a new Tokenizer from bytes

impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D>

where M: Serialize, N: Serialize, PT: Serialize, PP: Serialize, D: Serialize,

pub fn to_string(&self, pretty: bool) -> Result<String>

Serialize the current tokenizer as a String

pub fn save<P: AsRef<Path>>(&self, path: P, pretty: bool) -> Result<()>

Save the current tokenizer at the given path

Trait Implementations

impl<M: Clone, N: Clone, PT: Clone, PP: Clone, D: Clone> Clone for TokenizerImpl<M, N, PT, PP, D>

fn clone(&self) -> TokenizerImpl<M, N, PT, PP, D>

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<M: Debug, N: Debug, PT: Debug, PP: Debug, D: Debug> Debug for TokenizerImpl<M, N, PT, PP, D>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de, M, N, PT, PP, D> Deserialize<'de> for TokenizerImpl<M, N, PT, PP, D>

where M: Deserialize<'de> + Model, N: Deserialize<'de> + Normalizer, PT: Deserialize<'de> + PreTokenizer, PP: Deserialize<'de> + PostProcessor, D: Deserialize<'de> + Decoder,

fn deserialize<De>(deserializer: De) -> Result<Self, De::Error>

where De: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl<M, N, PT, PP, D> From<TokenizerImpl<M, N, PT, PP, D>> for Tokenizer

where M: Into<ModelWrapper>, N: Into<NormalizerWrapper>, PT: Into<PreTokenizerWrapper>, PP: Into<PostProcessorWrapper>, D: Into<DecoderWrapper>,

fn from(t: TokenizerImpl<M, N, PT, PP, D>) -> Self

Converts to this type from the input type.

impl<M, N, PT, PP, D> FromStr for TokenizerImpl<M, N, PT, PP, D>

where M: for<'de> Deserialize<'de> + Model, N: for<'de> Deserialize<'de> + Normalizer, PT: for<'de> Deserialize<'de> + PreTokenizer, PP: for<'de> Deserialize<'de> + PostProcessor, D: for<'de> Deserialize<'de> + Decoder,

type Err = Box<dyn Error + Send + Sync>

The associated error which can be returned from parsing.

fn from_str(s: &str) -> Result<Self>

Parses a string s to return a value of this type.

impl<M, N, PT, PP, D> Serialize for TokenizerImpl<M, N, PT, PP, D>

where M: Serialize, N: Serialize, PT: Serialize, PP: Serialize, D: Serialize,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.