Enum rust_bert::pipelines::common::TokenizerOption

pub enum TokenizerOption {
    Bert(BertTokenizer),
    Roberta(RobertaTokenizer),
    XLMRoberta(XLMRobertaTokenizer),
    Marian(MarianTokenizer),
    T5(T5Tokenizer),
    Albert(AlbertTokenizer),
}

Abstraction that holds a particular tokenizer, can be of any of the supported models

Variants

Bert(BertTokenizer)

Bert Tokenizer

Roberta(RobertaTokenizer)

Roberta Tokenizer

XLMRoberta(XLMRobertaTokenizer)

Roberta Tokenizer

Marian(MarianTokenizer)

Marian Tokenizer

T5(T5Tokenizer)

T5 Tokenizer

Albert(AlbertTokenizer)

Albert Tokenizer

Implementations

impl TokenizerOption

pub fn from_file(
    model_type: ModelType,
    vocab_path: &str,
    merges_path: Option<&str>,
    lower_case: bool
) -> Self

Interface method to load a tokenizer from file

pub fn model_type(&self) -> ModelType

Returns the model type

pub fn encode_list(
    &self,
    text_list: Vec<&str>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>

Interface method

pub fn tokenize(&self, text: &str) -> Vec<String>

Interface method to tokenization

pub fn build_input_with_special_tokens(
    &self,
    tokens_1: Vec<i64>,
    tokens_2: Option<Vec<i64>>,
    offsets_1: Vec<Option<Offset>>,
    offsets_2: Option<Vec<Option<Offset>>>,
    original_offsets_1: Vec<Vec<OffsetSize>>,
    original_offsets_2: Option<Vec<Vec<OffsetSize>>>,
    mask_1: Vec<Mask>,
    mask_2: Option<Vec<Mask>>
) -> (Vec<i64>, Vec<i8>, Vec<i8>, Vec<Option<Offset>>, Vec<Vec<OffsetSize>>, Vec<Mask>)

Interface method to build input with special tokens

pub fn convert_tokens_to_ids(&self, tokens: &Vec<String>) -> Vec<i64>

Interface method to convert tokens to ids

pub fn get_pad_id(&self) -> Option<i64>

Interface method

pub fn get_sep_id(&self) -> Option<i64>

Interface method