Enum rust_bert::pipelines::common::TokenizerOption

pub enum TokenizerOption {
    Bert(BertTokenizer),
    Roberta(RobertaTokenizer),
    XLMRoberta(XLMRobertaTokenizer),
    Marian(MarianTokenizer),
    T5(T5Tokenizer),
    Albert(AlbertTokenizer),
    XLNet(XLNetTokenizer),
    GPT2(Gpt2Tokenizer),
    OpenAiGpt(OpenAiGptTokenizer),
    Reformer(ReformerTokenizer),
}

Abstraction that holds a particular tokenizer, can be of any of the supported models

Variants

Bert(BertTokenizer)

Bert Tokenizer

Roberta(RobertaTokenizer)

Roberta Tokenizer

XLMRoberta(XLMRobertaTokenizer)

XLMRoberta Tokenizer

Marian(MarianTokenizer)

Marian Tokenizer

T5(T5Tokenizer)

T5 Tokenizer

Albert(AlbertTokenizer)

Albert Tokenizer

XLNet(XLNetTokenizer)

XLNet Tokenizer

GPT2(Gpt2Tokenizer)

GPT2 Tokenizer

OpenAiGpt(OpenAiGptTokenizer)

GPT Tokenizer

Reformer(ReformerTokenizer)

Reformer Tokenizer

Implementations

impl TokenizerOption

pub fn from_file(
    model_type: ModelType,
    vocab_path: &str,
    merges_path: Option<&str>,
    lower_case: bool,
    strip_accents: impl Into<Option<bool>>,
    add_prefix_space: impl Into<Option<bool>>
) -> Result<Self, RustBertError>

Interface method to load a tokenizer from file

pub fn model_type(&self) -> ModelType

Returns the model type

pub fn encode_list(
    &self,
    text_list: &[&str],
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>

Interface method

pub fn encode_pair_list(
    &self,
    text_pair_list: &[(&str, &str)],
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>

Interface method for pair encoding

pub fn tokenize(&self, text: &str) -> Vec<String>

Interface method to tokenization

pub fn tokenize_list(&self, text: &[&str]) -> Vec<Vec<String>>

Interface method to tokenization

pub fn decode(
    &self,
    token_ids: Vec<i64>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> String

Interface method to decoding

pub fn build_input_with_special_tokens(
    &self,
    token_ids_with_offsets_1: TokenIdsWithOffsets,
    token_ids_with_offsets_2: Option<TokenIdsWithOffsets>
) -> TokenizedInput

Interface method to build input with special tokens

pub fn convert_tokens_to_ids<S, ST>(&self, tokens: S) -> Vec<i64> where
    S: AsRef<[ST]>,
    ST: AsRef<str>, 

Interface method to convert tokens to ids

pub fn get_unk_id(&self) -> i64

Interface method

pub fn get_pad_id(&self) -> Option<i64>

Interface method

pub fn get_sep_id(&self) -> Option<i64>

Interface method