Enum rust_bert::pipelines::common::TokenizerOption

pub enum TokenizerOption {
    Bert(BertTokenizer),
    Roberta(RobertaTokenizer),
}

Abstraction that holds a particular tokenizer, can be of any of the supported models

Variants

Bert(BertTokenizer)

Bert Tokenizer

Roberta(RobertaTokenizer)

Roberta Tokenizer

Implementations

impl TokenizerOption

pub fn from_file(
    model_type: ModelType,
    vocab_path: &str,
    merges_path: Option<&str>,
    lower_case: bool
) -> Self

Interface method to load a tokenizer from file

pub fn model_type(&self) -> ModelType

Returns the model type

pub fn encode_list(
    &self,
    text_list: Vec<&str>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>

Interface method