Struct rust_tokenizers::preprocessing::tokenizer::base_tokenizer::TokenizedInput

pub struct TokenizedInput {
    pub token_ids: Vec<i64>,
    pub segment_ids: Vec<i8>,
    pub special_tokens_mask: Vec<i8>,
    pub overflowing_tokens: Vec<i64>,
    pub num_truncated_tokens: usize,
    pub token_offsets: Vec<Option<Offset>>,
    pub reference_offsets: Vec<Vec<OffsetSize>>,
    pub mask: Vec<Mask>,
}

Fields

token_ids: Vec<i64>

Vector of token IDs

segment_ids: Vec<i8>

Vector segments ids, segments are seperated with a [SEP] marker, each increments the segment ID. This vector has the same length as token_ids.

special_tokens_mask: Vec<i8>

Flags tokens as special tokens (1) or not (0). This vector has the same length as token_ids.

overflowing_tokens: Vec<i64>

num_truncated_tokens: usize

token_offsets: Vec<Option<Offset>>

Offset information in relation to the original text. Tokens that can not be related to the original source are registered as None.

reference_offsets: Vec<Vec<OffsetSize>>

mask: Vec<Mask>

Masks tokens so you can see what type of token something is. This vector has the same length as token_ids (and also makes special_tokens_mask redundant).

Trait Implementations

impl Clone for TokenizedInput

fn clone(&self) -> TokenizedInput

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenizedInput

fn fmt(&self, f: &mut Formatter) -> Result

Formats the value using the given formatter.

impl PartialEq<TokenizedInput> for TokenizedInput

fn eq(&self, other: &TokenizedInput) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &TokenizedInput) -> bool

This method tests for !=.

impl PartialOrd<TokenizedInput> for TokenizedInput

fn partial_cmp(&self, other: &TokenizedInput) -> Option<Ordering>

This method returns an ordering between self and other values if one exists.

fn lt(&self, other: &TokenizedInput) -> bool

This method tests less than (for self and other) and is used by the < operator.

fn le(&self, other: &TokenizedInput) -> bool

This method tests less than or equal to (for self and other) and is used by the <= operator.

fn gt(&self, other: &TokenizedInput) -> bool

This method tests greater than (for self and other) and is used by the > operator.

fn ge(&self, other: &TokenizedInput) -> bool

This method tests greater than or equal to (for self and other) and is used by the >= operator.

impl StructuralPartialEq for TokenizedInput