rusttorch-data 0.2.0

Typed datasets, samplers, batching, and loading for RustTorch
Documentation
use std::convert::Infallible;

use rusttorch_core::Tensor;
use rusttorch_data::{
    DataLoader, Dataset, DefaultCollator, LogicalSampleId, SequenceId, StreamDataLoaderBuilder,
    VecCollate, WorkerContext, WorkerRecord, WorkerSourceFactory,
};

struct Rows(usize);

impl Dataset for Rows {
    type Sample = Tensor;
    type Error = Infallible;

    fn len(&self) -> usize {
        self.0
    }

    fn get(&self, index: usize) -> Result<Self::Sample, Self::Error> {
        Ok(Tensor::from_slice(&[index as f32]))
    }
}

#[derive(Clone)]
struct Shards(usize);

impl WorkerSourceFactory for Shards {
    type Sample = usize;
    type Error = Infallible;
    type Source = std::vec::IntoIter<Result<WorkerRecord<usize>, Infallible>>;

    fn create(&self, worker: WorkerContext) -> Result<Self::Source, Self::Error> {
        Ok((worker.info.id..self.0)
            .step_by(worker.info.num_workers)
            .map(|index| {
                Ok(WorkerRecord {
                    sequence: Some(SequenceId::new(index as u64)),
                    logical_id: LogicalSampleId::new(index as u64),
                    sample: index,
                })
            })
            .collect::<Vec<_>>()
            .into_iter())
    }

    fn exact_len(&self) -> Option<usize> {
        Some(self.0)
    }
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut loader = DataLoader::builder(Rows(128))
        .shuffle(42)?
        .batch_size(64)
        .workers(4)
        .prefetch_factor(2)
        .collate(DefaultCollator)
        .pin_memory()
        .build()?;

    for batch in loader.iter() {
        let batch = batch?;
        assert_eq!(batch.size()[0], 64);
    }

    let mut stream = StreamDataLoaderBuilder::new(Shards(16))
        .workers(4)
        .batch_size(4)
        .collate(VecCollate)
        .build()?;
    let values = stream
        .iter()
        .collect::<Result<Vec<_>, _>>()?
        .into_iter()
        .flatten()
        .collect::<Vec<_>>();
    assert_eq!(values, (0..16).collect::<Vec<_>>());
    Ok(())
}