Skip to main content

incin_data/
loader.rs

1use crate::dataset::Dataset;
2use rand::seq::SliceRandom;
3use rand::thread_rng;
4use std::sync::mpsc::{Receiver, sync_channel};
5use std::sync::{Arc, Mutex};
6use std::thread;
7
8/// Collate.
9pub trait Collate<T>: Send + Sync {
10    /// Output.
11    type Output: Send + 'static;
12    /// Collate.
13    fn collate(&self, batch: Vec<T>) -> Self::Output;
14}
15
16/// Data loader.
17pub struct DataLoader<D, C>
18where
19    D: Dataset + 'static,
20    C: Collate<D::Item> + 'static,
21{
22    dataset: Arc<D>,
23    collate_fn: Arc<C>,
24    batch_size: usize,
25    num_workers: usize,
26    shuffle: bool,
27}
28
29impl<D, C> DataLoader<D, C>
30where
31    D: Dataset + 'static,
32    C: Collate<D::Item> + 'static,
33{
34    /// New.
35    pub fn new(dataset: D, collate_fn: C, batch_size: usize) -> Self {
36        Self {
37            dataset: Arc::new(dataset),
38            collate_fn: Arc::new(collate_fn),
39            batch_size,
40            num_workers: 0,
41            shuffle: false,
42        }
43    }
44
45    /// With num workers.
46    pub fn with_num_workers(mut self, num_workers: usize) -> Self {
47        self.num_workers = num_workers;
48        self
49    }
50
51    /// With shuffle.
52    pub fn with_shuffle(mut self, shuffle: bool) -> Self {
53        self.shuffle = shuffle;
54        self
55    }
56}
57
58/// Data loader iter.
59pub struct DataLoaderIter<T> {
60    receiver: Receiver<T>,
61}
62
63impl<T> Iterator for DataLoaderIter<T> {
64    /// Item.
65    type Item = T;
66
67    /// Next.
68    fn next(&mut self) -> Option<Self::Item> {
69        self.receiver.recv().ok()
70    }
71}
72
73impl<D, C> IntoIterator for &DataLoader<D, C>
74where
75    D: Dataset + 'static,
76    C: Collate<D::Item> + 'static,
77{
78    /// Item.
79    type Item = C::Output;
80    /// Into iter.
81    type IntoIter = DataLoaderIter<C::Output>;
82
83    /// Into iter.
84    fn into_iter(self) -> Self::IntoIter {
85        let mut indices: Vec<usize> = (0..self.dataset.len()).collect();
86        if self.shuffle {
87            indices.shuffle(&mut thread_rng());
88        }
89
90        let num_batches = indices.len().div_ceil(self.batch_size);
91        let mut batch_indices = Vec::with_capacity(num_batches);
92
93        for i in 0..num_batches {
94            let start = i * self.batch_size;
95            let end = core::cmp::min(start + self.batch_size, indices.len());
96            batch_indices.push(indices[start..end].to_vec());
97        }
98
99        // Bounded channel to prevent over-fetching
100        let (tx, rx) = sync_channel(self.num_workers * 2 + 2);
101
102        let dataset = self.dataset.clone();
103        let collate_fn = self.collate_fn.clone();
104
105        if self.num_workers == 0 {
106            // Single-threaded
107            thread::spawn(move || {
108                for batch_idx in batch_indices {
109                    let mut batch = Vec::with_capacity(batch_idx.len());
110                    for idx in batch_idx {
111                        if let Some(item) = dataset.get(idx) {
112                            batch.push(item);
113                        }
114                    }
115                    if !batch.is_empty() {
116                        let collated = collate_fn.collate(batch);
117                        if tx.send(collated).is_err() {
118                            break;
119                        }
120                    }
121                }
122            });
123        } else {
124            // Multi-threaded
125            let batch_indices = Arc::new(Mutex::new(batch_indices.into_iter()));
126
127            for _ in 0..self.num_workers {
128                let dataset = dataset.clone();
129                let collate_fn = collate_fn.clone();
130                let tx = tx.clone();
131                let batch_indices = batch_indices.clone();
132
133                thread::spawn(move || {
134                    loop {
135                        let next_batch = {
136                            let mut iter = batch_indices.lock().unwrap();
137                            iter.next()
138                        };
139
140                        if let Some(batch_idx) = next_batch {
141                            let mut batch = Vec::with_capacity(batch_idx.len());
142                            for idx in batch_idx {
143                                if let Some(item) = dataset.get(idx) {
144                                    batch.push(item);
145                                }
146                            }
147                            if !batch.is_empty() {
148                                let collated = collate_fn.collate(batch);
149                                if tx.send(collated).is_err() {
150                                    break;
151                                }
152                            }
153                        } else {
154                            break;
155                        }
156                    }
157                });
158            }
159        }
160
161        DataLoaderIter { receiver: rx }
162    }
163}
164
165#[cfg(test)]
166mod tests {
167    use super::*;
168    use std::collections::HashSet;
169    use std::time::Duration;
170
171    /// A trivial in-memory dataset of `i32`s, `0..len`.
172    struct RangeDataset(usize);
173
174    impl Dataset for RangeDataset {
175        type Item = i32;
176        fn len(&self) -> usize {
177            self.0
178        }
179        fn get(&self, index: usize) -> Option<i32> {
180            if index < self.0 {
181                Some(index as i32)
182            } else {
183                None
184            }
185        }
186    }
187
188    /// Collates a batch by just returning it as-is (`Vec<i32>`).
189    struct VecCollate;
190    impl Collate<i32> for VecCollate {
191        type Output = Vec<i32>;
192        fn collate(&self, batch: Vec<i32>) -> Vec<i32> {
193            batch
194        }
195    }
196
197    // Every test below has a hard wall-clock timeout: a regression in the
198    // worker-thread loop (e.g. a channel deadlock, an off-by-one in
199    // `batch_indices` that spins forever) must fail loudly and quickly
200    // instead of hanging the test suite indefinitely.
201    fn collect_with_timeout<T>(iter: impl Iterator<Item = T> + Send + 'static) -> Vec<T>
202    where
203        T: Send + 'static,
204    {
205        let (tx, rx) = std::sync::mpsc::channel();
206        std::thread::spawn(move || {
207            let _ = tx.send(iter.collect::<Vec<T>>());
208        });
209        rx.recv_timeout(Duration::from_secs(10))
210            .expect("DataLoader iteration did not finish within 10s (possible deadlock/hang)")
211    }
212
213    #[test]
214    fn single_threaded_yields_every_item_in_order_when_not_shuffled() {
215        let loader = DataLoader::new(RangeDataset(10), VecCollate, 3);
216        let batches = collect_with_timeout((&loader).into_iter());
217
218        // 10 items / batch_size 3 -> batches of [3,3,3,1], nothing dropped,
219        // nothing duplicated, original order preserved (no shuffle).
220        assert_eq!(
221            batches,
222            vec![vec![0, 1, 2], vec![3, 4, 5], vec![6, 7, 8], vec![9]]
223        );
224    }
225
226    #[test]
227    fn single_threaded_exact_division_produces_no_short_final_batch() {
228        let loader = DataLoader::new(RangeDataset(9), VecCollate, 3);
229        let batches = collect_with_timeout((&loader).into_iter());
230        assert_eq!(batches, vec![vec![0, 1, 2], vec![3, 4, 5], vec![6, 7, 8]]);
231    }
232
233    #[test]
234    fn empty_dataset_produces_zero_batches() {
235        let loader = DataLoader::new(RangeDataset(0), VecCollate, 4);
236        let batches = collect_with_timeout((&loader).into_iter());
237        assert!(batches.is_empty());
238    }
239
240    #[test]
241    fn batch_size_larger_than_dataset_produces_one_short_batch() {
242        let loader = DataLoader::new(RangeDataset(3), VecCollate, 100);
243        let batches = collect_with_timeout((&loader).into_iter());
244        assert_eq!(batches, vec![vec![0, 1, 2]]);
245    }
246
247    #[test]
248    fn shuffle_preserves_the_full_set_of_items_just_reorders_them() {
249        let loader = DataLoader::new(RangeDataset(50), VecCollate, 7).with_shuffle(true);
250        let batches = collect_with_timeout((&loader).into_iter());
251
252        let all_items: HashSet<i32> = batches.iter().flatten().copied().collect();
253        assert_eq!(
254            all_items,
255            (0..50).collect::<HashSet<_>>(),
256            "shuffling must not drop or duplicate any item"
257        );
258        // 50 / 7 = 7 batches of 7 plus one short batch of 1.
259        assert_eq!(batches.len(), 8);
260    }
261
262    #[test]
263    fn multi_worker_yields_every_item_exactly_once_no_duplicates_no_drops() {
264        // Regression guard for the DataLoader's core concurrency invariant:
265        // multiple worker threads pull batch-index chunks from a single
266        // Mutex<Iterator> (loader.rs's `batch_indices`) and push collated
267        // batches through a shared bounded mpsc channel. A bug here (e.g. two
268        // workers racing on the same chunk, or one silently starving) would
269        // show up as a missing or duplicated item — this is exactly the kind
270        // of concurrency bug that's invisible without a real multi-thread run.
271        let loader = DataLoader::new(RangeDataset(1000), VecCollate, 10).with_num_workers(8);
272        let batches = collect_with_timeout((&loader).into_iter());
273
274        let mut all_items: Vec<i32> = batches.into_iter().flatten().collect();
275        assert_eq!(
276            all_items.len(),
277            1000,
278            "no item should be dropped or duplicated"
279        );
280        all_items.sort_unstable();
281        assert_eq!(all_items, (0..1000).collect::<Vec<i32>>());
282    }
283
284    #[test]
285    fn multi_worker_with_shuffle_still_covers_every_item_exactly_once() {
286        let loader = DataLoader::new(RangeDataset(500), VecCollate, 6)
287            .with_num_workers(4)
288            .with_shuffle(true);
289        let batches = collect_with_timeout((&loader).into_iter());
290
291        let mut all_items: Vec<i32> = batches.into_iter().flatten().collect();
292        assert_eq!(all_items.len(), 500);
293        all_items.sort_unstable();
294        assert_eq!(all_items, (0..500).collect::<Vec<i32>>());
295    }
296
297    #[test]
298    fn more_workers_than_batches_does_not_hang_or_lose_items() {
299        // Edge case: 3 batches total but 8 workers requested. Workers that
300        // find the shared iterator already exhausted must exit cleanly
301        // instead of spinning or blocking the others.
302        let loader = DataLoader::new(RangeDataset(25), VecCollate, 10).with_num_workers(8);
303        let batches = collect_with_timeout((&loader).into_iter());
304
305        let mut all_items: Vec<i32> = batches.into_iter().flatten().collect();
306        assert_eq!(all_items.len(), 25);
307        all_items.sort_unstable();
308        assert_eq!(all_items, (0..25).collect::<Vec<i32>>());
309    }
310
311    #[test]
312    fn collate_function_is_actually_invoked_per_batch() {
313        // A Collate impl that transforms the batch (sums it) rather than
314        // passing it through unchanged, proving collate_fn is really called
315        // (not bypassed).
316        struct SumCollate;
317        impl Collate<i32> for SumCollate {
318            type Output = i32;
319            fn collate(&self, batch: Vec<i32>) -> i32 {
320                batch.into_iter().sum()
321            }
322        }
323
324        let loader = DataLoader::new(RangeDataset(6), SumCollate, 3);
325        let mut sums = collect_with_timeout((&loader).into_iter());
326        sums.sort_unstable();
327        // batch [0,1,2] -> 3, batch [3,4,5] -> 12
328        assert_eq!(sums, vec![3, 12]);
329    }
330}