Enum rust_bert::pipelines::common::TokenizerOption

source · [−]

pub enum TokenizerOption {
Show 18 variants
    Bert(BertTokenizer),
    Deberta(DeBERTaTokenizer),
    DebertaV2(DeBERTaV2Tokenizer),
    Roberta(RobertaTokenizer),
    XLMRoberta(XLMRobertaTokenizer),
    Marian(MarianTokenizer),
    T5(T5Tokenizer),
    Albert(AlbertTokenizer),
    XLNet(XLNetTokenizer),
    GPT2(Gpt2Tokenizer),
    OpenAiGpt(OpenAiGptTokenizer),
    Reformer(ReformerTokenizer),
    ProphetNet(ProphetNetTokenizer),
    Pegasus(PegasusTokenizer),
    MBart50(MBart50Tokenizer),
    M2M100(M2M100Tokenizer),
    FNet(FNetTokenizer),
    Bart(RobertaTokenizer),
}

Expand description

Abstraction that holds a particular tokenizer, can be of any of the supported models

Variants

`Bert(BertTokenizer)`

Bert Tokenizer

`Deberta(DeBERTaTokenizer)`

DeBERTa Tokenizer

`DebertaV2(DeBERTaV2Tokenizer)`

DeBERTa V2 Tokenizer

`Roberta(RobertaTokenizer)`

Roberta Tokenizer

`XLMRoberta(XLMRobertaTokenizer)`

XLMRoberta Tokenizer

`Marian(MarianTokenizer)`

Marian Tokenizer

`T5(T5Tokenizer)`

T5 Tokenizer

`Albert(AlbertTokenizer)`

Albert Tokenizer

`XLNet(XLNetTokenizer)`

XLNet Tokenizer

`GPT2(Gpt2Tokenizer)`

GPT2 Tokenizer

`OpenAiGpt(OpenAiGptTokenizer)`

GPT Tokenizer

`Reformer(ReformerTokenizer)`

Reformer Tokenizer

`ProphetNet(ProphetNetTokenizer)`

ProphetNet Tokenizer

`Pegasus(PegasusTokenizer)`

Pegasus Tokenizer

`MBart50(MBart50Tokenizer)`

MBart50 Tokenizer

`M2M100(M2M100Tokenizer)`

M2M100 Tokenizer

`FNet(FNetTokenizer)`

FNet Tokenizer

`Bart(RobertaTokenizer)`

Bart Tokenizer

Implementations

impl TokenizerOption

pub fn from_file(
    model_type: ModelType,
    vocab_path: &str,
    merges_path: Option<&str>,
    lower_case: bool,
    strip_accents: impl Into<Option<bool>>,
    add_prefix_space: impl Into<Option<bool>>
) -> Result<Self, RustBertError>

Interface method to load a tokenizer from file

pub fn model_type(&self) -> ModelType

Returns the model type

pub fn encode_list<S>(
    &self,
    text_list: &[S],
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A> where     A: Allocator,` where
    S: AsRef<str> + Sync,

Interface method

pub fn encode_pair_list(
    &self,
    text_pair_list: &[(&str, &str )],
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A> where     A: Allocator,`

Interface method for pair encoding

pub fn encode_pair(
    &self,
    text_1: &str,
    text_2: Option<&str>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> TokenizedInput

Interface method for pair encoding (single input)

pub fn tokenize(&self, text: &str) -> Vec<String>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A> where A: Allocator,`

Interface method to tokenization

pub fn tokenize_with_offsets(&self, text: &str) -> TokensWithOffsets

Interface method to tokenization

pub fn tokenize_list<S>(&self, text: &[S]) -> Vec<Vec<String>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A> where A: Allocator,` where
S: AsRef<str> + Sync,

Interface method to tokenization

pub fn decode(
    &self,
    token_ids: &[i64 ],
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> String

Interface method to decoding

pub fn build_input_with_special_tokens(
    &self,
    token_ids_with_offsets_1: TokenIdsWithOffsets,
    token_ids_with_offsets_2: Option<TokenIdsWithOffsets>
) -> TokenizedInput

Interface method to build input with special tokens

pub fn convert_tokens_to_ids<S>(&self, tokens: &[S]) -> Vec<i64>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A> where A: Allocator,` where
S: AsRef<str>,

Interface method to convert tokens to ids

pub fn get_unk_id(&self) -> i64

Interface method

pub fn get_pad_id(&self) -> Option<i64>

Interface method

pub fn get_sep_id(&self) -> Option<i64>

Interface method

pub fn get_bos_id(&self) -> Option<i64>

Interface method

pub fn get_eos_id(&self) -> Option<i64>

Interface method

Auto Trait Implementations

impl RefUnwindSafe for TokenizerOption

impl Send for TokenizerOption

impl Sync for TokenizerOption

impl Unpin for TokenizerOption

impl UnwindSafe for TokenizerOption

Blanket Implementations

impl<T> Any for T where
T: 'static + ?Sized,

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

impl<T> Borrow<T> for T where
T: ?Sized,

const: unstable · source

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

impl<T> BorrowMut<T> for T where
T: ?Sized,

const: unstable · source

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

impl<T> From<T> for T

const: unstable · source

fn from(t: T) -> T

Returns the argument unchanged.

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

Instruments this type with the provided Span, returning an Instrumented wrapper. Read more

fn in_current_span(self) -> Instrumented<Self>

Instruments this type with the current Span, returning an Instrumented wrapper. Read more

impl<T, U> Into<U> for T where
U: From<T>,

const: unstable · source

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

impl<T> Pointable for T

const ALIGN: usize = mem::align_of::()

The alignment of pointer.

type Init = T

The type for initializers.

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more

impl<T> Same<T> for T

type Output = T

Should always be Self

impl<T, U> TryFrom<U> for T where
U: Into<T>,

type Error = Infallible

The type returned in the event of a conversion error.

const: unstable · source

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

Performs the conversion.

impl<T, U> TryInto<U> for T where
U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.

const: unstable · source

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.

impl<V, T> VZip<V> for T where
V: MultiLane<T>,

fn vzip(self) -> V

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> where
S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a WithDispatch wrapper. Read more

fn with_current_subscriber(self) -> WithDispatch<Self>

Attaches the current default Subscriber to this type, returning a WithDispatch wrapper. Read more