Struct rust_tokenizers::preprocessing::tokenizer::bert_tokenizer::BertTokenizer

pub struct BertTokenizer { /* fields omitted */ }

Methods

impl BertTokenizer

pub fn from_file(path: &str, lower_case: bool) -> BertTokenizer

pub fn from_existing_vocab(
    vocab: Arc<BertVocab>,
    lower_case: bool
) -> BertTokenizer

Trait Implementations

impl MultiThreadedTokenizer<BertVocab> for BertTokenizer

fn vocab(&self) -> &T

fn tokenize_list(&self, text_list: Vec<&str>) -> Vec<Vec<String>>

fn encode_list(
    &self,
    text_list: Vec<&str>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>

fn encode_pair_list(
    &self,
    text_list: Vec<(&str, &str)>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>

fn decode_list(
    &self,
    token_ids_list: Vec<Vec<i64>>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> Vec<String>

impl Tokenizer<BertVocab> for BertTokenizer

fn vocab(&self) -> &BertVocab

fn tokenize(&self, text: &str) -> Vec<String>

fn build_input_with_special_tokens(
    &self,
    tokens_1: Vec<i64>,
    tokens_2: Option<Vec<i64>>
) -> (Vec<i64>, Vec<i8>, Vec<i8>)

fn convert_tokens_to_string(&self, tokens: Vec<String>) -> String

fn tokenize_list(&self, text_list: Vec<&str>) -> Vec<Vec<String>>

fn convert_tokens_to_ids(&self, tokens: &Vec<String>) -> Vec<i64>

fn encode(
    &self,
    text_1: &str,
    text_2: Option<&str>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> TokenizedInput

fn encode_list(
    &self,
    text_list: Vec<&str>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>

fn encode_pair_list(
    &self,
    text_list: Vec<(&str, &str)>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>

fn decode(
    &self,
    token_ids: Vec<i64>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> String

fn clean_up_tokenization(&self, input_string: String) -> String

fn decode_list(
    &self,
    token_ids_list: Vec<Vec<i64>>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> Vec<String>