1use crate::dataset::Dataset;
2use rand::seq::SliceRandom;
3use rand::thread_rng;
4use std::sync::mpsc::{Receiver, sync_channel};
5use std::sync::{Arc, Mutex};
6use std::thread;
7
8pub trait Collate<T>: Send + Sync {
10 type Output: Send + 'static;
12 fn collate(&self, batch: Vec<T>) -> Self::Output;
14}
15
16pub struct DataLoader<D, C>
18where
19 D: Dataset + 'static,
20 C: Collate<D::Item> + 'static,
21{
22 dataset: Arc<D>,
23 collate_fn: Arc<C>,
24 batch_size: usize,
25 num_workers: usize,
26 shuffle: bool,
27}
28
29impl<D, C> DataLoader<D, C>
30where
31 D: Dataset + 'static,
32 C: Collate<D::Item> + 'static,
33{
34 pub fn new(dataset: D, collate_fn: C, batch_size: usize) -> Self {
36 Self {
37 dataset: Arc::new(dataset),
38 collate_fn: Arc::new(collate_fn),
39 batch_size,
40 num_workers: 0,
41 shuffle: false,
42 }
43 }
44
45 pub fn with_num_workers(mut self, num_workers: usize) -> Self {
47 self.num_workers = num_workers;
48 self
49 }
50
51 pub fn with_shuffle(mut self, shuffle: bool) -> Self {
53 self.shuffle = shuffle;
54 self
55 }
56}
57
58pub struct DataLoaderIter<T> {
60 receiver: Receiver<T>,
61}
62
63impl<T> Iterator for DataLoaderIter<T> {
64 type Item = T;
66
67 fn next(&mut self) -> Option<Self::Item> {
69 self.receiver.recv().ok()
70 }
71}
72
73impl<D, C> IntoIterator for &DataLoader<D, C>
74where
75 D: Dataset + 'static,
76 C: Collate<D::Item> + 'static,
77{
78 type Item = C::Output;
80 type IntoIter = DataLoaderIter<C::Output>;
82
83 fn into_iter(self) -> Self::IntoIter {
85 let mut indices: Vec<usize> = (0..self.dataset.len()).collect();
86 if self.shuffle {
87 indices.shuffle(&mut thread_rng());
88 }
89
90 let num_batches = indices.len().div_ceil(self.batch_size);
91 let mut batch_indices = Vec::with_capacity(num_batches);
92
93 for i in 0..num_batches {
94 let start = i * self.batch_size;
95 let end = core::cmp::min(start + self.batch_size, indices.len());
96 batch_indices.push(indices[start..end].to_vec());
97 }
98
99 let (tx, rx) = sync_channel(self.num_workers * 2 + 2);
101
102 let dataset = self.dataset.clone();
103 let collate_fn = self.collate_fn.clone();
104
105 if self.num_workers == 0 {
106 thread::spawn(move || {
108 for batch_idx in batch_indices {
109 let mut batch = Vec::with_capacity(batch_idx.len());
110 for idx in batch_idx {
111 if let Some(item) = dataset.get(idx) {
112 batch.push(item);
113 }
114 }
115 if !batch.is_empty() {
116 let collated = collate_fn.collate(batch);
117 if tx.send(collated).is_err() {
118 break;
119 }
120 }
121 }
122 });
123 } else {
124 let batch_indices = Arc::new(Mutex::new(batch_indices.into_iter()));
126
127 for _ in 0..self.num_workers {
128 let dataset = dataset.clone();
129 let collate_fn = collate_fn.clone();
130 let tx = tx.clone();
131 let batch_indices = batch_indices.clone();
132
133 thread::spawn(move || {
134 loop {
135 let next_batch = {
136 let mut iter = batch_indices.lock().unwrap();
137 iter.next()
138 };
139
140 if let Some(batch_idx) = next_batch {
141 let mut batch = Vec::with_capacity(batch_idx.len());
142 for idx in batch_idx {
143 if let Some(item) = dataset.get(idx) {
144 batch.push(item);
145 }
146 }
147 if !batch.is_empty() {
148 let collated = collate_fn.collate(batch);
149 if tx.send(collated).is_err() {
150 break;
151 }
152 }
153 } else {
154 break;
155 }
156 }
157 });
158 }
159 }
160
161 DataLoaderIter { receiver: rx }
162 }
163}
164
165#[cfg(test)]
166mod tests {
167 use super::*;
168 use std::collections::HashSet;
169 use std::time::Duration;
170
171 struct RangeDataset(usize);
173
174 impl Dataset for RangeDataset {
175 type Item = i32;
176 fn len(&self) -> usize {
177 self.0
178 }
179 fn get(&self, index: usize) -> Option<i32> {
180 if index < self.0 {
181 Some(index as i32)
182 } else {
183 None
184 }
185 }
186 }
187
188 struct VecCollate;
190 impl Collate<i32> for VecCollate {
191 type Output = Vec<i32>;
192 fn collate(&self, batch: Vec<i32>) -> Vec<i32> {
193 batch
194 }
195 }
196
197 fn collect_with_timeout<T>(iter: impl Iterator<Item = T> + Send + 'static) -> Vec<T>
202 where
203 T: Send + 'static,
204 {
205 let (tx, rx) = std::sync::mpsc::channel();
206 std::thread::spawn(move || {
207 let _ = tx.send(iter.collect::<Vec<T>>());
208 });
209 rx.recv_timeout(Duration::from_secs(10))
210 .expect("DataLoader iteration did not finish within 10s (possible deadlock/hang)")
211 }
212
213 #[test]
214 fn single_threaded_yields_every_item_in_order_when_not_shuffled() {
215 let loader = DataLoader::new(RangeDataset(10), VecCollate, 3);
216 let batches = collect_with_timeout((&loader).into_iter());
217
218 assert_eq!(
221 batches,
222 vec![vec![0, 1, 2], vec![3, 4, 5], vec![6, 7, 8], vec![9]]
223 );
224 }
225
226 #[test]
227 fn single_threaded_exact_division_produces_no_short_final_batch() {
228 let loader = DataLoader::new(RangeDataset(9), VecCollate, 3);
229 let batches = collect_with_timeout((&loader).into_iter());
230 assert_eq!(batches, vec![vec![0, 1, 2], vec![3, 4, 5], vec![6, 7, 8]]);
231 }
232
233 #[test]
234 fn empty_dataset_produces_zero_batches() {
235 let loader = DataLoader::new(RangeDataset(0), VecCollate, 4);
236 let batches = collect_with_timeout((&loader).into_iter());
237 assert!(batches.is_empty());
238 }
239
240 #[test]
241 fn batch_size_larger_than_dataset_produces_one_short_batch() {
242 let loader = DataLoader::new(RangeDataset(3), VecCollate, 100);
243 let batches = collect_with_timeout((&loader).into_iter());
244 assert_eq!(batches, vec![vec![0, 1, 2]]);
245 }
246
247 #[test]
248 fn shuffle_preserves_the_full_set_of_items_just_reorders_them() {
249 let loader = DataLoader::new(RangeDataset(50), VecCollate, 7).with_shuffle(true);
250 let batches = collect_with_timeout((&loader).into_iter());
251
252 let all_items: HashSet<i32> = batches.iter().flatten().copied().collect();
253 assert_eq!(
254 all_items,
255 (0..50).collect::<HashSet<_>>(),
256 "shuffling must not drop or duplicate any item"
257 );
258 assert_eq!(batches.len(), 8);
260 }
261
262 #[test]
263 fn multi_worker_yields_every_item_exactly_once_no_duplicates_no_drops() {
264 let loader = DataLoader::new(RangeDataset(1000), VecCollate, 10).with_num_workers(8);
272 let batches = collect_with_timeout((&loader).into_iter());
273
274 let mut all_items: Vec<i32> = batches.into_iter().flatten().collect();
275 assert_eq!(
276 all_items.len(),
277 1000,
278 "no item should be dropped or duplicated"
279 );
280 all_items.sort_unstable();
281 assert_eq!(all_items, (0..1000).collect::<Vec<i32>>());
282 }
283
284 #[test]
285 fn multi_worker_with_shuffle_still_covers_every_item_exactly_once() {
286 let loader = DataLoader::new(RangeDataset(500), VecCollate, 6)
287 .with_num_workers(4)
288 .with_shuffle(true);
289 let batches = collect_with_timeout((&loader).into_iter());
290
291 let mut all_items: Vec<i32> = batches.into_iter().flatten().collect();
292 assert_eq!(all_items.len(), 500);
293 all_items.sort_unstable();
294 assert_eq!(all_items, (0..500).collect::<Vec<i32>>());
295 }
296
297 #[test]
298 fn more_workers_than_batches_does_not_hang_or_lose_items() {
299 let loader = DataLoader::new(RangeDataset(25), VecCollate, 10).with_num_workers(8);
303 let batches = collect_with_timeout((&loader).into_iter());
304
305 let mut all_items: Vec<i32> = batches.into_iter().flatten().collect();
306 assert_eq!(all_items.len(), 25);
307 all_items.sort_unstable();
308 assert_eq!(all_items, (0..25).collect::<Vec<i32>>());
309 }
310
311 #[test]
312 fn collate_function_is_actually_invoked_per_batch() {
313 struct SumCollate;
317 impl Collate<i32> for SumCollate {
318 type Output = i32;
319 fn collate(&self, batch: Vec<i32>) -> i32 {
320 batch.into_iter().sum()
321 }
322 }
323
324 let loader = DataLoader::new(RangeDataset(6), SumCollate, 3);
325 let mut sums = collect_with_timeout((&loader).into_iter());
326 sums.sort_unstable();
327 assert_eq!(sums, vec![3, 12]);
329 }
330}