Struct BatchedTokenizedInput 

pub struct BatchedTokenizedInput {
    pub input_ids: Vec<Vec<u32>>,
    pub attention_mask: Vec<Vec<u8>>,
    pub token_type_ids: Option<Vec<Vec<u32>>>,
}

Batched tokenized input with convenient access methods

Fields

input_ids: Vec<Vec<u32>>

attention_mask: Vec<Vec<u8>>

token_type_ids: Option<Vec<Vec<u32>>>

Implementations

impl BatchedTokenizedInput

pub fn from_batch(batch: Vec<TokenizedInput>) -> Self

Create from a batch of TokenizedInput

pub fn batch_size(&self) -> usize

Get the batch size

pub fn sequence_lengths(&self) -> Vec<usize>

Get the sequence length for each sample

pub fn to_individual(self) -> Vec<TokenizedInput>

Convert to individual TokenizedInput items

pub fn input_ids_tensor(&self) -> &Vec<Vec<u32>>

Get input IDs as a flat tensor-like structure

pub fn attention_mask_tensor(&self) -> &Vec<Vec<u8>>

Get attention mask as a flat tensor-like structure

pub fn token_type_ids_tensor(&self) -> Option<&Vec<Vec<u32>>>

Get token type IDs as a flat tensor-like structure

Trait Implementations

impl Clone for BatchedTokenizedInput

fn clone(&self) -> BatchedTokenizedInput

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for BatchedTokenizedInput

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.