entrenar 0.7.12

Training & Optimization library with autograd, LoRA, quantization, and model merging
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

//! Dataset example

/// A single example from a dataset
#[derive(Debug, Clone)]
pub struct Example {
    /// Input token IDs
    pub input_ids: Vec<u32>,
    /// Attention mask (1 = attend, 0 = ignore)
    pub attention_mask: Vec<u8>,
    /// Target labels (for supervised learning)
    pub labels: Option<Vec<u32>>,
    /// Text content (if available)
    pub text: Option<String>,
}

impl Example {
    /// Create new example from token IDs
    #[must_use]
    pub fn from_tokens(input_ids: Vec<u32>) -> Self {
        let len = input_ids.len();
        Self { input_ids, attention_mask: vec![1; len], labels: None, text: None }
    }

    /// Set labels
    #[must_use]
    pub fn with_labels(mut self, labels: Vec<u32>) -> Self {
        self.labels = Some(labels);
        self
    }

    /// Set text
    #[must_use]
    pub fn with_text(mut self, text: impl Into<String>) -> Self {
        self.text = Some(text.into());
        self
    }

    /// Get sequence length
    #[must_use]
    pub fn len(&self) -> usize {
        self.input_ids.len()
    }

    /// Check if empty
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.input_ids.is_empty()
    }
}