pub trait Dataset {
type Sample;
type Error;
// Required methods
fn len(&self) -> usize;
fn get(&self, index: usize) -> Result<Self::Sample, Self::Error>;
// Provided methods
fn get_batch(
&self,
indices: &[usize],
) -> Result<Vec<Self::Sample>, Self::Error> { ... }
fn get_batch_with_context(
&self,
indices: &[usize],
context: &WorkerContext,
) -> Result<Vec<Self::Sample>, Self::Error> { ... }
fn is_empty(&self) -> bool { ... }
fn samples(&self) -> DatasetSamples<'_, Self> ⓘ
where Self: Sized { ... }
}Expand description
A finite, indexable collection of samples.
Implementations return owned samples so loaders can move them into batches
without cloning. Dataset-specific failures remain in Dataset::Error.
Keep len() stable during an iteration and make each sampled index valid
for get(). Override Self::get_batch when one file read or database query
can retrieve multiple rows more efficiently.
§Return typed feature/label pairs
The default collator stacks tensor features and converts integer labels into a tensor. Ordinary Rust storage can be shared by workers even though each fetched tensor is owned by a single sample.
use std::convert::Infallible;
use rusttorch_core::Tensor;
use rusttorch_data::{DataLoader, Dataset};
struct Rows(Vec<([f32; 2], i64)>);
impl Dataset for Rows {
type Sample = (Tensor, i64);
type Error = Infallible;
fn len(&self) -> usize { self.0.len() }
fn get(&self, index: usize) -> Result<Self::Sample, Infallible> {
let (values, label) = &self.0[index];
Ok((Tensor::from_slice(values), *label))
}
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let rows = Rows(vec![([1., 2.], 0), ([3., 4.], 1)]);
let mut loader = DataLoader::builder(rows).batch_size(2).workers(2).build()?;
let (features, labels) = loader.iter().next().unwrap()?;
assert_eq!(features.size(), [2, 2]);
assert_eq!(Vec::<i64>::try_from(&labels)?, [0, 1]);
Ok(())
}Required Associated Types§
Required Methods§
Provided Methods§
Sourcefn get_batch(&self, indices: &[usize]) -> Result<Vec<Self::Sample>, Self::Error>
fn get_batch(&self, indices: &[usize]) -> Result<Vec<Self::Sample>, Self::Error>
Loads samples for one index batch.
The default calls Dataset::get once per index. Overrides must
return exactly one sample per supplied index and preserve index order.
Sourcefn get_batch_with_context(
&self,
indices: &[usize],
context: &WorkerContext,
) -> Result<Vec<Self::Sample>, Self::Error>
fn get_batch_with_context( &self, indices: &[usize], context: &WorkerContext, ) -> Result<Vec<Self::Sample>, Self::Error>
Loads one index batch with cooperative worker lifecycle context.
The source-compatible default ignores context and delegates to
Dataset::get_batch exactly once. Context-aware datasets may
override this hook to honor cancellation and deadlines between bounded
operations. Rust cannot force-cancel an arbitrary blocking native call.
Sourcefn samples(&self) -> DatasetSamples<'_, Self> ⓘwhere
Self: Sized,
fn samples(&self) -> DatasetSamples<'_, Self> ⓘwhere
Self: Sized,
Iterates over every sample in index order.
The iterator borrows the dataset and calls Dataset::get lazily.
Dyn Compatibility§
This trait is dyn compatible.
In older versions of Rust, dyn compatibility was called "object safety".