Struct DataExample 

pub struct DataExample {
    pub input_ids: Vec<u32>,
    pub attention_mask: Option<Vec<u32>>,
    pub token_type_ids: Option<Vec<u32>>,
    pub labels: Option<Vec<i64>>,
    pub metadata: HashMap<String, Value>,
}

Input example for data collation

Represents a single training/inference example that can be batched together with other examples during data collation.

Fields

input_ids: Vec<u32>

Token IDs for the input sequence

attention_mask: Option<Vec<u32>>

Attention mask indicating which tokens should be attended to

token_type_ids: Option<Vec<u32>>

Token type IDs for distinguishing different segments

labels: Option<Vec<i64>>

Labels for supervised learning tasks

metadata: HashMap<String, Value>

Additional metadata for the example

Implementations

impl DataExample

pub fn new(input_ids: Vec<u32>) -> Self

Create a new data example with basic input IDs

pub fn sequence_length(&self) -> usize

Get the sequence length

pub fn with_attention_mask(self, mask: Vec<u32>) -> Self

Add an attention mask

pub fn with_token_type_ids(self, type_ids: Vec<u32>) -> Self

Add token type IDs

pub fn with_labels(self, labels: Vec<i64>) -> Self

Add labels

pub fn with_metadata(self, key: String, value: Value) -> Self

Add metadata

pub fn has_labels(&self) -> bool

Check if the example has labels

Trait Implementations

impl Clone for DataExample

fn clone(&self) -> DataExample

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for DataExample

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for DataExample

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for DataExample

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.