Struct TokenizedInput 

pub struct TokenizedInput {
    pub input_ids: Vec<usize>,
    pub attention_mask: Vec<f32>,
    pub token_type_ids: Option<Vec<usize>>,
    pub position_ids: Option<Vec<usize>>,
}

Tokenized input representation

Fields

input_ids: Vec<usize>

Token IDs

attention_mask: Vec<f32>

Attention mask

token_type_ids: Option<Vec<usize>>

Token type IDs (for BERT-style models)

position_ids: Option<Vec<usize>>

Position IDs

Trait Implementations

impl Clone for TokenizedInput

fn clone(&self) -> TokenizedInput

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenizedInput

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.