rust_transformers::preprocessing::tokenizer::base_tokenizer

Trait MultiThreadedTokenizer

Source 
pub trait MultiThreadedTokenizer<T: Vocab>
where
    Self: Sync + Send + Tokenizer<T>,
{
    // Provided methods
    fn tokenize_list(&self, text_list: Vec<&str>) -> Vec<Vec<String>> { ... }
    fn encode_list(
        &self,
        text_list: Vec<&str>,
        max_len: usize,
        truncation_strategy: &TruncationStrategy,
        stride: usize,
    ) -> Vec<TokenizedInput> { ... }
    fn encode_pair_list(
        &self,
        text_list: Vec<(&str, &str)>,
        max_len: usize,
        truncation_strategy: &TruncationStrategy,
        stride: usize,
    ) -> Vec<TokenizedInput> { ... }
}

Provided Methods§

Source

fn tokenize_list(&self, text_list: Vec<&str>) -> Vec<Vec<String>>

Source

fn encode_list( &self, text_list: Vec<&str>, max_len: usize, truncation_strategy: &TruncationStrategy, stride: usize, ) -> Vec<TokenizedInput>

Source

fn encode_pair_list( &self, text_list: Vec<(&str, &str)>, max_len: usize, truncation_strategy: &TruncationStrategy, stride: usize, ) -> Vec<TokenizedInput>

Implementors§