Skip to main content

mathverse_ai/
data.rs

1//! Data loading utilities: DataLoader with batching and shuffling.
2
3use crate::tensor::Tensor;
4use mathverse_core::error::{MathError, MathResult};
5
6/// A mini-batch of inputs and targets.
7pub struct Batch {
8    /// Batch inputs (first dimension is batch size).
9    pub x: Tensor,
10    /// Batch targets (first dimension is batch size).
11    pub y: Tensor,
12}
13
14/// DataLoader: iterates over data in shuffled mini-batches.
15pub struct DataLoader {
16    /// Full input tensor (first dimension is dataset size).
17    pub x: Tensor,
18    /// Full target tensor (first dimension is dataset size).
19    pub y: Tensor,
20    /// Number of samples per batch.
21    pub batch_size: usize,
22    /// Whether to shuffle indices on `reset()`.
23    pub shuffle: bool,
24    /// Seed for the shuffle RNG.
25    pub seed: u64,
26    indices: Vec<usize>,
27    pos: usize,
28}
29
30impl DataLoader {
31    /// Create a new [`DataLoader`].
32    ///
33    /// # Panics
34    ///
35    /// Panics if `x.shape[0]` is not present (i.e. `x` is scalar).
36    pub fn new(x: Tensor, y: Tensor, batch_size: usize, shuffle: bool) -> Self {
37        let n = x.shape[0];
38        let indices: Vec<usize> = (0..n).collect();
39        Self { x, y, batch_size, shuffle, seed: 0xABCD, indices, pos: 0 }
40    }
41
42    /// Create a new [`DataLoader`] with a specific shuffle seed.
43    pub fn with_seed(x: Tensor, y: Tensor, batch_size: usize, shuffle: bool, seed: u64) -> Self {
44        let n = x.shape[0];
45        let indices: Vec<usize> = (0..n).collect();
46        Self { x, y, batch_size, shuffle, seed, indices, pos: 0 }
47    }
48
49    /// Reset iterator. Uses the loader's seed for deterministic shuffling.
50    pub fn reset(&mut self) {
51        self.pos = 0;
52        if self.shuffle {
53            let mut state = self.seed;
54            // Fisher-Yates shuffle with xorshift
55            for i in (1..self.indices.len()).rev() {
56                state ^= state << 13; state ^= state >> 7; state ^= state << 17;
57                let j = (state as usize) % (i + 1);
58                self.indices.swap(i, j);
59            }
60        }
61    }
62
63    /// Number of batches.
64    pub fn num_batches(&self) -> usize {
65        self.x.shape[0].div_ceil(self.batch_size)
66    }
67
68    /// Get next batch.
69    pub fn next_batch(&mut self) -> Option<Batch> {
70        if self.pos >= self.x.shape[0] { return None; }
71        let end = (self.pos + self.batch_size).min(self.x.shape[0]);
72        let batch_len = end - self.pos;
73        let dim: usize = self.x.shape[1..].iter().product();
74        let y_dim: usize = if self.y.shape.len() > 1 { self.y.shape[1..].iter().product() } else { 1 };
75
76        let mut x_data = Vec::with_capacity(batch_len * dim);
77        let mut y_data = Vec::with_capacity(batch_len * y_dim);
78        for &idx in &self.indices[self.pos..end] {
79            x_data.extend_from_slice(&self.x.data[idx * dim..(idx + 1) * dim]);
80            y_data.extend_from_slice(&self.y.data[idx * y_dim..(idx + 1) * y_dim]);
81        }
82
83        let mut x_shape = self.x.shape.clone();
84        x_shape[0] = batch_len;
85        let mut y_shape = self.y.shape.clone();
86        y_shape[0] = batch_len;
87
88        self.pos = end;
89        Some(Batch {
90            x: Tensor { shape: x_shape, data: x_data },
91            y: Tensor { shape: y_shape, data: y_data },
92        })
93    }
94}
95
96/// Iterator adapter for DataLoader.
97impl Iterator for DataLoader {
98    type Item = Batch;
99
100    fn next(&mut self) -> Option<Self::Item> {
101        self.next_batch()
102    }
103}
104
105/// Train/test split: returns (x_train, x_test, y_train, y_test).
106///
107/// `test_ratio` should be in `[0.0, 1.0]`. This function is deterministic given `seed`.
108///
109/// # Errors
110///
111/// Returns `MathError::DimensionMismatch` if `x` and `y` have different
112/// numbers of samples, or if `test_ratio` is outside `[0.0, 1.0]`.
113pub fn train_test_split(x: &Tensor, y: &Tensor, test_ratio: f64, seed: u64) -> MathResult<(Tensor, Tensor, Tensor, Tensor)> {
114    if x.shape[0] != y.shape[0] { return Err(MathError::DimensionMismatch); }
115    if !(0.0..=1.0).contains(&test_ratio) { return Err(MathError::InvalidArgument("train_test_split: test_ratio must be in [0.0, 1.0]")); }
116    let n = x.shape[0];
117    let test_size = (n as f64 * test_ratio) as usize;
118    let train_size = n - test_size;
119
120    let mut indices: Vec<usize> = (0..n).collect();
121    // Deterministic shuffle
122    let mut state = seed;
123    for i in (1..n).rev() {
124        state ^= state << 13; state ^= state >> 7; state ^= state << 17;
125        let j = (state as usize) % (i + 1);
126        indices.swap(i, j);
127    }
128
129    let x_dim: usize = x.shape[1..].iter().product();
130    let y_dim: usize = if y.shape.len() > 1 { y.shape[1..].iter().product() } else { 1 };
131
132    let mut x_train_data = Vec::with_capacity(train_size * x_dim);
133    let mut x_test_data = Vec::with_capacity(test_size * x_dim);
134    let mut y_train_data = Vec::with_capacity(train_size * y_dim);
135    let mut y_test_data = Vec::with_capacity(test_size * y_dim);
136
137    for (i, &idx) in indices.iter().enumerate() {
138        if i < train_size {
139            x_train_data.extend_from_slice(&x.data[idx * x_dim..(idx + 1) * x_dim]);
140            y_train_data.extend_from_slice(&y.data[idx * y_dim..(idx + 1) * y_dim]);
141        } else {
142            x_test_data.extend_from_slice(&x.data[idx * x_dim..(idx + 1) * x_dim]);
143            y_test_data.extend_from_slice(&y.data[idx * y_dim..(idx + 1) * y_dim]);
144        }
145    }
146
147    let mut x_shape = x.shape.clone();
148    let mut y_shape = y.shape.clone();
149    x_shape[0] = train_size;
150    y_shape[0] = train_size;
151    let x_train = Tensor { shape: x_shape, data: x_train_data };
152    let y_train = Tensor { shape: y_shape, data: y_train_data };
153
154    let mut x_shape = x.shape.clone();
155    let mut y_shape = y.shape.clone();
156    x_shape[0] = test_size;
157    y_shape[0] = test_size;
158    let x_test = Tensor { shape: x_shape, data: x_test_data };
159    let y_test = Tensor { shape: y_shape, data: y_test_data };
160
161    Ok((x_train, x_test, y_train, y_test))
162}
163
164#[cfg(test)]
165mod tests {
166    use super::*;
167
168    #[test]
169    fn dataloader_test() {
170        let x = Tensor::arange(0.0, 10.0, 1.0).unwrap().reshape(&[10, 1]).unwrap();
171        let y = Tensor::arange(0.0, 10.0, 1.0).unwrap().reshape(&[10, 1]).unwrap();
172        let mut loader = DataLoader::new(x, y, 3, false);
173        assert_eq!(loader.num_batches(), 4);
174        let batch = loader.next().unwrap();
175        assert_eq!(batch.x.shape, vec![3, 1]);
176    }
177
178    #[test]
179    fn train_test_split_test() {
180        let x = Tensor::arange(0.0, 10.0, 1.0).unwrap().reshape(&[10, 1]).unwrap();
181        let y = Tensor::arange(0.0, 10.0, 1.0).unwrap().reshape(&[10, 1]).unwrap();
182        let (x_tr, x_te, _y_tr, _y_te) = train_test_split(&x, &y, 0.2, 42).unwrap();
183        assert_eq!(x_tr.shape[0], 8);
184        assert_eq!(x_te.shape[0], 2);
185        let mismatched = Tensor::arange(0.0, 5.0, 1.0).unwrap().reshape(&[5, 1]).unwrap();
186        assert!(train_test_split(&x, &mismatched, 0.2, 42).is_err());
187    }
188}
189
190