use std::convert::Infallible;
use rusttorch_core::Tensor;
use rusttorch_data::{
DataLoader, Dataset, DefaultCollator, DistributedSampler, LogicalSampleId, SequenceId,
StreamDataLoaderBuilder, VecCollate, WorkerContext, WorkerRecord, WorkerSourceFactory,
};
struct Rows(usize);
impl Dataset for Rows {
type Sample = Tensor;
type Error = Infallible;
fn len(&self) -> usize {
self.0
}
fn get(&self, index: usize) -> Result<Self::Sample, Self::Error> {
Ok(Tensor::from_slice(&[index as f32]))
}
}
struct Shards(usize);
struct RowShard {
next: usize,
end: usize,
stride: usize,
}
impl Iterator for RowShard {
type Item = Result<WorkerRecord<usize>, Infallible>;
fn next(&mut self) -> Option<Self::Item> {
if self.next >= self.end {
return None;
}
let index = self.next;
self.next += self.stride;
Some(Ok(WorkerRecord {
sequence: Some(SequenceId::new(index as u64)),
logical_id: LogicalSampleId::new(index as u64),
sample: index,
}))
}
}
impl WorkerSourceFactory for Shards {
type Sample = usize;
type Error = Infallible;
type Source = RowShard;
fn create(&self, worker: WorkerContext) -> Result<Self::Source, Self::Error> {
Ok(RowShard {
next: worker.info.id,
end: self.0,
stride: worker.info.num_workers,
})
}
fn exact_len(&self) -> Option<usize> {
Some(self.0)
}
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut loader = DataLoader::builder(Rows(128))
.shuffle(42)?
.batch_size(64)
.workers(4)
.prefetch_factor(2)
.collate(DefaultCollator)
.pin_memory()
.build()?;
for batch in loader.iter() {
let batch = batch?;
assert_eq!(batch.size()[0], 64);
}
let rank = 0;
let replicas = 2;
let sampler = DistributedSampler::new(128, replicas, rank, true, 42, false)?;
let mut distributed = DataLoader::builder(Rows(128))
.sampler(sampler)
.rank(rank)
.batch_size(32)
.build()?;
for epoch in 0..2 {
distributed.set_epoch(epoch);
for batch in distributed.iter() {
assert_eq!(batch?.size(), [32, 1]);
}
}
let mut stream = StreamDataLoaderBuilder::new(Shards(17))
.workers(4)
.batch_size(4)
.collate(VecCollate)
.build()?;
let batches = stream.iter().collect::<Result<Vec<_>, _>>()?;
assert_eq!(batches.last().unwrap(), &[16]);
let values = batches.into_iter().flatten().collect::<Vec<_>>();
assert_eq!(values, (0..17).collect::<Vec<_>>());
Ok(())
}