use crate::dataset::Dataset;
use rand::seq::SliceRandom;
use rand::thread_rng;
use std::sync::mpsc::{Receiver, sync_channel};
use std::sync::{Arc, Mutex};
use std::thread;
pub trait Collate<T>: Send + Sync {
type Output: Send + 'static;
fn collate(&self, batch: Vec<T>) -> Self::Output;
}
pub struct DataLoader<D, C>
where
D: Dataset + 'static,
C: Collate<D::Item> + 'static,
{
dataset: Arc<D>,
collate_fn: Arc<C>,
batch_size: usize,
num_workers: usize,
shuffle: bool,
}
impl<D, C> DataLoader<D, C>
where
D: Dataset + 'static,
C: Collate<D::Item> + 'static,
{
pub fn new(dataset: D, collate_fn: C, batch_size: usize) -> Self {
Self {
dataset: Arc::new(dataset),
collate_fn: Arc::new(collate_fn),
batch_size,
num_workers: 0,
shuffle: false,
}
}
pub fn with_num_workers(mut self, num_workers: usize) -> Self {
self.num_workers = num_workers;
self
}
pub fn with_shuffle(mut self, shuffle: bool) -> Self {
self.shuffle = shuffle;
self
}
}
pub struct DataLoaderIter<T> {
receiver: Receiver<T>,
}
impl<T> Iterator for DataLoaderIter<T> {
type Item = T;
fn next(&mut self) -> Option<Self::Item> {
self.receiver.recv().ok()
}
}
impl<D, C> IntoIterator for &DataLoader<D, C>
where
D: Dataset + 'static,
C: Collate<D::Item> + 'static,
{
type Item = C::Output;
type IntoIter = DataLoaderIter<C::Output>;
fn into_iter(self) -> Self::IntoIter {
let mut indices: Vec<usize> = (0..self.dataset.len()).collect();
if self.shuffle {
indices.shuffle(&mut thread_rng());
}
let num_batches = indices.len().div_ceil(self.batch_size);
let mut batch_indices = Vec::with_capacity(num_batches);
for i in 0..num_batches {
let start = i * self.batch_size;
let end = core::cmp::min(start + self.batch_size, indices.len());
batch_indices.push(indices[start..end].to_vec());
}
let (tx, rx) = sync_channel(self.num_workers * 2 + 2);
let dataset = self.dataset.clone();
let collate_fn = self.collate_fn.clone();
if self.num_workers == 0 {
thread::spawn(move || {
for batch_idx in batch_indices {
let mut batch = Vec::with_capacity(batch_idx.len());
for idx in batch_idx {
if let Some(item) = dataset.get(idx) {
batch.push(item);
}
}
if !batch.is_empty() {
let collated = collate_fn.collate(batch);
if tx.send(collated).is_err() {
break;
}
}
}
});
} else {
let batch_indices = Arc::new(Mutex::new(batch_indices.into_iter()));
for _ in 0..self.num_workers {
let dataset = dataset.clone();
let collate_fn = collate_fn.clone();
let tx = tx.clone();
let batch_indices = batch_indices.clone();
thread::spawn(move || {
loop {
let next_batch = {
let mut iter = batch_indices.lock().unwrap();
iter.next()
};
if let Some(batch_idx) = next_batch {
let mut batch = Vec::with_capacity(batch_idx.len());
for idx in batch_idx {
if let Some(item) = dataset.get(idx) {
batch.push(item);
}
}
if !batch.is_empty() {
let collated = collate_fn.collate(batch);
if tx.send(collated).is_err() {
break;
}
}
} else {
break;
}
}
});
}
}
DataLoaderIter { receiver: rx }
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashSet;
use std::time::Duration;
struct RangeDataset(usize);
impl Dataset for RangeDataset {
type Item = i32;
fn len(&self) -> usize {
self.0
}
fn get(&self, index: usize) -> Option<i32> {
if index < self.0 {
Some(index as i32)
} else {
None
}
}
}
struct VecCollate;
impl Collate<i32> for VecCollate {
type Output = Vec<i32>;
fn collate(&self, batch: Vec<i32>) -> Vec<i32> {
batch
}
}
fn collect_with_timeout<T>(iter: impl Iterator<Item = T> + Send + 'static) -> Vec<T>
where
T: Send + 'static,
{
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let _ = tx.send(iter.collect::<Vec<T>>());
});
rx.recv_timeout(Duration::from_secs(10))
.expect("DataLoader iteration did not finish within 10s (possible deadlock/hang)")
}
#[test]
fn single_threaded_yields_every_item_in_order_when_not_shuffled() {
let loader = DataLoader::new(RangeDataset(10), VecCollate, 3);
let batches = collect_with_timeout((&loader).into_iter());
assert_eq!(
batches,
vec![vec![0, 1, 2], vec![3, 4, 5], vec![6, 7, 8], vec![9]]
);
}
#[test]
fn single_threaded_exact_division_produces_no_short_final_batch() {
let loader = DataLoader::new(RangeDataset(9), VecCollate, 3);
let batches = collect_with_timeout((&loader).into_iter());
assert_eq!(batches, vec![vec![0, 1, 2], vec![3, 4, 5], vec![6, 7, 8]]);
}
#[test]
fn empty_dataset_produces_zero_batches() {
let loader = DataLoader::new(RangeDataset(0), VecCollate, 4);
let batches = collect_with_timeout((&loader).into_iter());
assert!(batches.is_empty());
}
#[test]
fn batch_size_larger_than_dataset_produces_one_short_batch() {
let loader = DataLoader::new(RangeDataset(3), VecCollate, 100);
let batches = collect_with_timeout((&loader).into_iter());
assert_eq!(batches, vec![vec![0, 1, 2]]);
}
#[test]
fn shuffle_preserves_the_full_set_of_items_just_reorders_them() {
let loader = DataLoader::new(RangeDataset(50), VecCollate, 7).with_shuffle(true);
let batches = collect_with_timeout((&loader).into_iter());
let all_items: HashSet<i32> = batches.iter().flatten().copied().collect();
assert_eq!(
all_items,
(0..50).collect::<HashSet<_>>(),
"shuffling must not drop or duplicate any item"
);
assert_eq!(batches.len(), 8);
}
#[test]
fn multi_worker_yields_every_item_exactly_once_no_duplicates_no_drops() {
let loader = DataLoader::new(RangeDataset(1000), VecCollate, 10).with_num_workers(8);
let batches = collect_with_timeout((&loader).into_iter());
let mut all_items: Vec<i32> = batches.into_iter().flatten().collect();
assert_eq!(
all_items.len(),
1000,
"no item should be dropped or duplicated"
);
all_items.sort_unstable();
assert_eq!(all_items, (0..1000).collect::<Vec<i32>>());
}
#[test]
fn multi_worker_with_shuffle_still_covers_every_item_exactly_once() {
let loader = DataLoader::new(RangeDataset(500), VecCollate, 6)
.with_num_workers(4)
.with_shuffle(true);
let batches = collect_with_timeout((&loader).into_iter());
let mut all_items: Vec<i32> = batches.into_iter().flatten().collect();
assert_eq!(all_items.len(), 500);
all_items.sort_unstable();
assert_eq!(all_items, (0..500).collect::<Vec<i32>>());
}
#[test]
fn more_workers_than_batches_does_not_hang_or_lose_items() {
let loader = DataLoader::new(RangeDataset(25), VecCollate, 10).with_num_workers(8);
let batches = collect_with_timeout((&loader).into_iter());
let mut all_items: Vec<i32> = batches.into_iter().flatten().collect();
assert_eq!(all_items.len(), 25);
all_items.sort_unstable();
assert_eq!(all_items, (0..25).collect::<Vec<i32>>());
}
#[test]
fn collate_function_is_actually_invoked_per_batch() {
struct SumCollate;
impl Collate<i32> for SumCollate {
type Output = i32;
fn collate(&self, batch: Vec<i32>) -> i32 {
batch.into_iter().sum()
}
}
let loader = DataLoader::new(RangeDataset(6), SumCollate, 3);
let mut sums = collect_with_timeout((&loader).into_iter());
sums.sort_unstable();
assert_eq!(sums, vec![3, 12]);
}
}