Struct BaseTokenizer

pub struct BaseTokenizer<T: Vocab> { /* private fields */ }

Implementations

impl<T: Vocab + Sync + Send> BaseTokenizer<T>

pub fn from_file(path: &str) -> BaseTokenizer<T>

pub fn from_existing_vocab(vocab: Arc<T>) -> BaseTokenizer<T>

Trait Implementations

impl<T: Vocab + Sync + Send> MultiThreadedTokenizer<T> for BaseTokenizer<T>

fn tokenize_list(&self, text_list: Vec<&str>) -> Vec<Vec<String>>

fn encode_list( &self, text_list: Vec<&str>, max_len: usize, truncation_strategy: &TruncationStrategy, stride: usize, ) -> Vec<TokenizedInput>

fn encode_pair_list( &self, text_list: Vec<(&str, &str)>, max_len: usize, truncation_strategy: &TruncationStrategy, stride: usize, ) -> Vec<TokenizedInput>

impl<T: Vocab + Sync + Send> Tokenizer<T> for BaseTokenizer<T>

fn vocab(&self) -> &T

fn tokenize(&self, text: &str) -> Vec<String>

fn tokenize_list(&self, text_list: Vec<&str>) -> Vec<Vec<String>>

fn convert_tokens_to_ids(&self, tokens: &Vec<String>) -> Vec<i64>

fn encode( &self, text_1: &str, text_2: Option<&str>, max_len: usize, truncation_strategy: &TruncationStrategy, stride: usize, ) -> TokenizedInput

fn encode_list( &self, text_list: Vec<&str>, max_len: usize, truncation_strategy: &TruncationStrategy, stride: usize, ) -> Vec<TokenizedInput>

fn encode_pair_list( &self, text_list: Vec<(&str, &str)>, max_len: usize, truncation_strategy: &TruncationStrategy, stride: usize, ) -> Vec<TokenizedInput>

fn build_input_with_special_tokens( &self, tokens_1: Vec<i64>, tokens_2: Option<Vec<i64>>, ) -> (Vec<i64>, Vec<i8>, Vec<i8>)