use dataloader_rs::collator::{Collator, DefaultCollator};
use dataloader_rs::{DataLoader, Dataset, error::Result};
type Point = (u32, [f32; 2]);
struct Points;
impl Dataset for Points {
type Item = Point;
fn get(&self, index: usize) -> Result<Point> {
let x = index as f32;
Ok((index as u32 % 2, [x, x * x]))
}
fn len(&self) -> usize {
10
}
}
#[derive(Debug)]
struct Features {
labels: Vec<u32>,
values: Vec<f32>,
shape: [usize; 2],
}
struct ToFeatures;
impl Collator<Point> for ToFeatures {
type Batch = Features;
fn collate(&self, items: Vec<Point>) -> Result<Features> {
let shape = [items.len(), 2];
let mut labels = Vec::with_capacity(items.len());
let mut values = Vec::with_capacity(items.len() * 2);
for (label, point) in items {
labels.push(label);
values.extend(point);
}
Ok(Features {
labels,
values,
shape,
})
}
}
fn main() -> Result<()> {
let mut loader = DataLoader::builder(Points).batch_size(4).build();
let batch: Vec<Point> = loader.iter().next().expect("a batch")?;
println!("VecCollator: {batch:?}");
let mut loader = DataLoader::builder(Points)
.batch_size(4)
.collator(DefaultCollator)
.build();
let (labels, [xs, ys]) = loader.iter().next().expect("a batch")?;
println!("DefaultCollator: labels {labels:?}, xs {xs:?}, ys {ys:?}");
let mut loader = DataLoader::builder(Points)
.batch_size(4)
.num_workers(2)
.collator(ToFeatures)
.build();
for batch in &mut loader {
let batch = batch?;
println!(
"ToFeatures: shape {:?}, labels {:?}, values {:?}",
batch.shape, batch.labels, batch.values
);
}
Ok(())
}