Enum rust_bert::pipelines::common::TokenizerOption

pub enum TokenizerOption {
    Bert(BertTokenizer),
    Roberta(RobertaTokenizer),
    XLMRoberta(XLMRobertaTokenizer),
    Marian(MarianTokenizer),
    T5(T5Tokenizer),
    Albert(AlbertTokenizer),
    XLNet(XLNetTokenizer),
    GPT2(Gpt2Tokenizer),
    OpenAiGpt(OpenAiGptTokenizer),
    Reformer(ReformerTokenizer),
    ProphetNet(ProphetNetTokenizer),
    Pegasus(PegasusTokenizer),
    MBart50(MBart50Tokenizer),
    M2M100(M2M100Tokenizer),
    FNet(FNetTokenizer),
}

Abstraction that holds a particular tokenizer, can be of any of the supported models

Variants

Bert(BertTokenizer)

Tuple Fields

0: BertTokenizer

Bert Tokenizer

Roberta(RobertaTokenizer)

Tuple Fields

0: RobertaTokenizer

Roberta Tokenizer

XLMRoberta(XLMRobertaTokenizer)

Tuple Fields

0: XLMRobertaTokenizer

XLMRoberta Tokenizer

Marian(MarianTokenizer)

Tuple Fields

0: MarianTokenizer

Marian Tokenizer

T5(T5Tokenizer)

Tuple Fields

0: T5Tokenizer

T5 Tokenizer

Albert(AlbertTokenizer)

Tuple Fields

0: AlbertTokenizer

Albert Tokenizer

XLNet(XLNetTokenizer)

Tuple Fields

0: XLNetTokenizer

XLNet Tokenizer

GPT2(Gpt2Tokenizer)

Tuple Fields

0: Gpt2Tokenizer

GPT2 Tokenizer

OpenAiGpt(OpenAiGptTokenizer)

Tuple Fields

0: OpenAiGptTokenizer

GPT Tokenizer

Reformer(ReformerTokenizer)

Tuple Fields

0: ReformerTokenizer

Reformer Tokenizer

ProphetNet(ProphetNetTokenizer)

Tuple Fields

0: ProphetNetTokenizer

ProphetNet Tokenizer

Pegasus(PegasusTokenizer)

Tuple Fields

0: PegasusTokenizer

Pegasus Tokenizer

MBart50(MBart50Tokenizer)

Tuple Fields

0: MBart50Tokenizer

MBart50 Tokenizer

M2M100(M2M100Tokenizer)

Tuple Fields

0: M2M100Tokenizer

M2M100 Tokenizer

FNet(FNetTokenizer)

Tuple Fields

0: FNetTokenizer

FNet Tokenizer

Implementations

impl TokenizerOption

pub fn from_file(
    model_type: ModelType,
    vocab_path: &str,
    merges_path: Option<&str>,
    lower_case: bool,
    strip_accents: impl Into<Option<bool>>,
    add_prefix_space: impl Into<Option<bool>>
) -> Result<Self, RustBertError>

Interface method to load a tokenizer from file

pub fn model_type(&self) -> ModelType

Returns the model type

pub fn encode_list<S>(
    &self,
    text_list: &[S],
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput> where
    S: AsRef<str> + Sync, 

Interface method

pub fn encode_pair_list(
    &self,
    text_pair_list: &[(&str, &str)],
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>

Interface method for pair encoding

pub fn encode_pair(
    &self,
    text_1: &str,
    text_2: Option<&str>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> TokenizedInput

Interface method for pair encoding (single input)

pub fn tokenize(&self, text: &str) -> Vec<String>

Interface method to tokenization

pub fn tokenize_with_offsets(&self, text: &str) -> TokensWithOffsets

Interface method to tokenization

pub fn tokenize_list<S>(&self, text: &[S]) -> Vec<Vec<String>> where
    S: AsRef<str> + Sync, 

Interface method to tokenization

pub fn decode(
    &self,
    token_ids: &[i64],
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> String

Interface method to decoding

pub fn build_input_with_special_tokens(
    &self,
    token_ids_with_offsets_1: TokenIdsWithOffsets,
    token_ids_with_offsets_2: Option<TokenIdsWithOffsets>
) -> TokenizedInput

Interface method to build input with special tokens

pub fn convert_tokens_to_ids<S>(&self, tokens: &[S]) -> Vec<i64> where
    S: AsRef<str>, 

Interface method to convert tokens to ids

pub fn get_unk_id(&self) -> i64

Interface method

pub fn get_pad_id(&self) -> Option<i64>

Interface method

pub fn get_sep_id(&self) -> Option<i64>

Interface method