pub struct DataLoaderBuilder<D, P, C, F = IdentityTransformFactory, I = NoWorkerInit, X = SerialExecution, M = MemoryDisabled, N = PinDisabled, K = CheckpointDisabled> { /* private fields */ }Expand description
Configure repeated passes over an owned indexed dataset.
Create this with crate::DataLoader::builder. By default, samples arrive
in index order, batch size is one, the final short batch is kept, and work
runs on the calling thread. crate::DefaultCollator stacks tensors and
numeric values recursively while preserving tuple fields.
§Shuffle each epoch and prepare batches with workers
Keep immutable metadata in the dataset and do decoding in get(). A positive
worker count overlaps fetching and per-sample transforms; collation remains
on the calling thread. The example uses small Rust records so its sampling
and ownership behavior is visible without a decoder.
use std::convert::Infallible;
use rusttorch_data::{DataLoader, Dataset, VecCollate};
struct Rows;
impl Dataset for Rows {
type Sample = i64;
type Error = Infallible;
fn len(&self) -> usize { 6 }
fn get(&self, index: usize) -> Result<i64, Infallible> { Ok(index as i64) }
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut loader = DataLoader::builder(Rows)
.batch_size(2).shuffle(42)?.workers(2).prefetch_factor(2)
.collate(VecCollate).build()?;
for epoch in 0..3 {
loader.set_epoch(epoch);
let batches = loader.iter().collect::<Result<Vec<_>, _>>()?;
assert_eq!(batches.len(), 3);
let mut visited: Vec<_> = batches.into_iter().flatten().collect();
visited.sort_unstable();
assert_eq!(visited, [0, 1, 2, 3, 4, 5]);
}
Ok(())
}Repeating an epoch repeats its sampling order. Self::shuffle sets the
sampler seed; Self::seed separately seeds worker and sample-transform
contexts. Omit .workers(...) for datasets that are not Send + Sync,
including crate::TensorDataset. Calling .workers(0) still selects the
worker-safe API and requires those bounds.
§Pad variable-length sequences in a custom batch
crate::FnCollate receives the selected samples after transforms. Return
your model’s batch representation, including lengths or padding masks when
needed. Here zero is the padding value and lengths identify real tokens.
use std::convert::Infallible;
use rusttorch_core::Tensor;
use rusttorch_data::{DataLoader, Dataset, FnCollate};
struct Sentences(Vec<Vec<i64>>);
impl Dataset for Sentences {
type Sample = Vec<i64>;
type Error = Infallible;
fn len(&self) -> usize { self.0.len() }
fn get(&self, index: usize) -> Result<Self::Sample, Infallible> {
Ok(self.0[index].clone())
}
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let padding = FnCollate::new(|rows: Vec<Vec<i64>>| {
let width = rows.iter().map(Vec::len).max().unwrap_or(0);
let lengths: Vec<_> = rows.iter().map(|row| row.len() as i64).collect();
let count = rows.len();
let mut values = Vec::new();
for mut row in rows {
row.resize(width, 0);
values.extend(row);
}
Ok::<_, rusttorch_core::RustTorchError>((
Tensor::f_from_slice(&values)?.f_reshape([count as i64, width as i64])?,
Tensor::f_from_slice(&lengths)?,
))
});
let mut loader = DataLoader::builder(Sentences(vec![vec![1, 2], vec![3, 4, 5]]))
.batch_size(2).collate(padding).build()?;
let (tokens, lengths) = loader.iter().next().unwrap()?;
assert_eq!(Vec::<Vec<i64>>::try_from(&tokens)?, [vec![1, 2, 0], vec![3, 4, 5]]);
assert_eq!(Vec::<i64>::try_from(&lengths)?, [2, 3]);
Ok(())
}§Infer one sample at a time
Self::without_batching uses conversion instead of stacking. It adds no
batch dimension and is useful when each dataset item is already a model
input or a preassembled batch. Do not also set batch_size or drop_last.
use rusttorch_core::Tensor;
use rusttorch_data::{DataLoader, TensorDataset};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let features = Tensor::from_slice(&[1_f32, 2., 3., 4.]).reshape([2, 2]);
let mut loader = DataLoader::builder(TensorDataset::new(vec![features])?)
.without_batching().build()?;
for sample in loader.iter() {
assert_eq!(sample?[0].size(), [2]);
}
Ok(())
}Use Self::dataset_identity with a replay-safe dataset to enable exact
restart; crate::LoaderState has a complete checkpoint example. Independent
stream shards use crate::StreamDataLoaderBuilder instead.
Implementations§
Source§impl<D, P, C, F, N> DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, CheckpointFresh>where
D: WorkerReplayDataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint + Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: LoaderPlanConfiguration + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, N> DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, CheckpointFresh>where
D: WorkerReplayDataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint + Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: LoaderPlanConfiguration + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>>
pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>>
Builds an exact replay loader with explicit immutable dataset evidence.
Positive workers require ordered delivery, no timeout or persistence, disabled byte accounting, and no worker initializer. Snapshot heap contents are item-bounded, not measured as total resident bytes.
Source§impl<D, P, C, F, N, SS, TS, CS> DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, CheckpointResume<LoaderState<(), SS, WorkerTransformState<TS>, CS>>>where
D: WorkerReplayDataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint<State = TS> + Send + 'static,
TS: Clone + Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: LoaderPlanConfiguration + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C, State = SS, CoordinatorState = CS>,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, N, SS, TS, CS> DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, CheckpointResume<LoaderState<(), SS, WorkerTransformState<TS>, CS>>>where
D: WorkerReplayDataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint<State = TS> + Send + 'static,
TS: Clone + Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: LoaderPlanConfiguration + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C, State = SS, CoordinatorState = CS>,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>, CheckpointBuildError<F::Error>>
pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>, CheckpointBuildError<F::Error>>
Validates every component before restoring retained worker instances.
Source§impl<D> DataLoaderBuilder<D, AutoBatch<SequentialSampler>, DefaultCollator>where
D: Dataset,
impl<D> DataLoaderBuilder<D, AutoBatch<SequentialSampler>, DefaultCollator>where
D: Dataset,
Source§impl<D, P, C, F, I, X, M, N, K> DataLoaderBuilder<D, P, C, F, I, X, M, N, K>
impl<D, P, C, F, I, X, M, N, K> DataLoaderBuilder<D, P, C, F, I, X, M, N, K>
Sourcepub fn batch_size(self, batch_size: usize) -> Self
pub fn batch_size(self, batch_size: usize) -> Self
Sets how many selected samples form one automatic batch; defaults to one.
Zero is rejected by build(). The final batch may be smaller unless
Self::drop_last is enabled. This option cannot be combined with an
explicit batch sampler or Self::without_batching.
Sourcepub fn drop_last(self, drop_last: bool) -> Self
pub fn drop_last(self, drop_last: bool) -> Self
Omits the final incomplete automatic batch when true; defaults to false.
Use this when a training step requires a fixed batch size. It does not change how a distributed sampler divides indices between ranks.
Sourcepub fn workers(
self,
workers: usize,
) -> DataLoaderBuilder<D, P, C, F, I, WorkerExecution, M, N, K>
pub fn workers( self, workers: usize, ) -> DataLoaderBuilder<D, P, C, F, I, WorkerExecution, M, N, K>
Selects bounded map-worker execution and its thread-safe capability.
Positive counts share the map dataset through Arc, fetch and
transform in deterministic worker lanes, and collate on the calling
thread. Passing zero keeps serial execution, but selecting this method
still requires worker-safe types at compile time.
See the DataLoaderBuilder epoch example for a complete pipeline.
Sourcepub fn seed(self, seed: u64) -> Self
pub fn seed(self, seed: u64) -> Self
Seeds deterministic worker initialization and sample-transform contexts.
This does not enable shuffling or replace the sampler’s seed. Select
Self::shuffle or a seeded Self::sampler separately.
Sourcepub fn epoch(self, epoch: u64) -> Self
pub fn epoch(self, epoch: u64) -> Self
Sets the initial epoch used by sampling and task contexts.
Sourcepub fn rank(self, rank: usize) -> Self
pub fn rank(self, rank: usize) -> Self
Sets the distributed rank included in deterministic contexts.
This does not partition the dataset. Use crate::DistributedSampler
with the same rank to select its indices; gradient synchronization belongs
to the training application.
Sourcepub fn transform<T>(
self,
transform: T,
) -> DataLoaderBuilder<D, P, C, CloneTransformFactory<T>, I, X, M, N, K>
pub fn transform<T>( self, transform: T, ) -> DataLoaderBuilder<D, P, C, CloneTransformFactory<T>, I, X, M, N, K>
Clones transform once for each serial iterator or worker lifecycle.
Sourcepub fn transform_factory<F2>(
self,
factory: F2,
) -> DataLoaderBuilder<D, P, C, F2, I, X, M, N, K>
pub fn transform_factory<F2>( self, factory: F2, ) -> DataLoaderBuilder<D, P, C, F2, I, X, M, N, K>
Replaces transform construction with an explicit typed factory.
Sourcepub fn checkpoint_stateless(
self,
) -> DataLoaderBuilder<D, P, C, StatelessTransformFactory<F>, I, X, M, N, K>
pub fn checkpoint_stateless( self, ) -> DataLoaderBuilder<D, P, C, StatelessTransformFactory<F>, I, X, M, N, K>
Marks the current factory’s transform as explicitly stateless for exact checkpointing.
Sourcepub fn checkpoint_transactional(
self,
) -> DataLoaderBuilder<D, P, C, TransactionalTransformFactory<F>, I, X, M, N, K>
pub fn checkpoint_transactional( self, ) -> DataLoaderBuilder<D, P, C, TransactionalTransformFactory<F>, I, X, M, N, K>
Marks the current factory’s transform as transactional for exact checkpointing.
Sourcepub fn worker_init<I2>(
self,
worker_init: I2,
) -> DataLoaderBuilder<D, P, C, F, I2, X, M, N, K>
pub fn worker_init<I2>( self, worker_init: I2, ) -> DataLoaderBuilder<D, P, C, F, I2, X, M, N, K>
Stores the initializer used by future positive-worker execution.
Sourcepub fn persistent_workers(self, persistent: bool) -> Self
pub fn persistent_workers(self, persistent: bool) -> Self
Selects whether positive-worker iterators reuse loader-owned threads.
Reuse avoids rebuilding expensive worker-local decoders each epoch.
Workers keep their initial crate::WorkerInfo seeds and factory-created
state. Dropping an iterator cancels and waits for its active work; the
threads remain available until the loader is dropped and joins the pool.
Sourcepub fn timeout(self, timeout: Duration) -> Self
pub fn timeout(self, timeout: Duration) -> Self
Sets the cooperative timeout for each blocking Iterator::next call.
Duration::ZERO disables the timeout. Expiry cancels the current
generation, drains it to quiescence, yields one typed timeout error, and
then ends the iterator. User dataset and transform code must observe its
crate::WorkerContext or crate::TaskContext to stop promptly;
Rust threads are never force-cancelled. A nonzero timeout that exceeds
the platform monotonic clock range is rejected by Self::build.
Sourcepub fn ordered(self, ordered: bool) -> Self
pub fn ordered(self, ordered: bool) -> Self
Keeps sampler order when true (the default), or yields ready batches first.
Completion order can reduce waits for a slow batch but changes the order observed by training. Exact checkpointing requires ordered delivery.
Sourcepub fn in_order(self, in_order: bool) -> Self
pub fn in_order(self, in_order: bool) -> Self
Selects sampler-order or completion-order delivery.
Sourcepub fn pin_memory(
self,
) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Auto>, K>
pub fn pin_memory( self, ) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Auto>, K>
Enables automatic recursive pinning for CUDA device zero when available.
Pinning happens after collation and preserves the batch type. It prepares
host memory for transfer; the application still moves tensors to its
model’s device. Without CUDA, batches pass through unchanged and
OwnedDataLoader::pin_memory_status reports that pinning is disabled.
Enabling pinning requires the final batch type to implement
PinMemory at build time:
use std::convert::Infallible;
use rusttorch_data::{DataLoader, Dataset, FnCollate};
struct Rows;
struct Batch;
impl Dataset for Rows {
type Sample = u8;
type Error = Infallible;
fn len(&self) -> usize { 1 }
fn get(&self, _: usize) -> Result<u8, Infallible> { Ok(1) }
}
let _ = DataLoader::builder(Rows)
.collate(FnCollate::new(|_: Vec<u8>| Ok::<_, Infallible>(Batch)))
.pin_memory()
.build();Sourcepub fn pin_memory_for(
self,
device: Device,
) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Explicit>, K>
pub fn pin_memory_for( self, device: Device, ) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Explicit>, K>
Enables recursive pinning for one explicit, available CUDA device.
Sourcepub fn prefetch_factor(self, factor: usize) -> Self
pub fn prefetch_factor(self, factor: usize) -> Self
Bounds outstanding batches per worker; defaults to two with positive workers.
For example, four workers and factor two permit eight outstanding batches.
Increase this when input preparation cannot otherwise keep training busy;
larger values retain more data. The unit here is batches, while
crate::StreamDataLoaderBuilder::prefetch_factor counts records.
The worker count multiplied by this factor is checked before any worker starts and is also the global outstanding-work credit limit. Capacities that overflow crossbeam’s ring arithmetic or whose conservative aggregate of concrete task/control/completion slots, worker bookkeeping, and channel control-block allowances exceeds the checked 64 MiB queue-allocation ceiling are rejected during build.
Sourcepub fn prefetch_bytes(
self,
limit: NonZeroUsize,
) -> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, N, K>
pub fn prefetch_bytes( self, limit: NonZeroUsize, ) -> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, N, K>
Enables a nonzero byte budget for final post-transform prefetched data.
The coordinator’s active collation batch is outside this queue budget and remains bounded by the configured batch size.
Enabling the budget requires the final transformed type to implement
MemoryFootprint at build time:
use std::{convert::Infallible, num::NonZeroUsize};
use rusttorch_data::{DataLoader, Dataset, VecCollate};
struct Sample;
struct Rows;
impl Dataset for Rows {
type Sample = Sample;
type Error = Infallible;
fn len(&self) -> usize { 1 }
fn get(&self, _: usize) -> Result<Sample, Infallible> { Ok(Sample) }
}
let _ = DataLoader::builder(Rows)
.workers(1)
.collate(VecCollate)
.prefetch_bytes(NonZeroUsize::MIN)
.build();Sourcepub fn batch_sampler<B>(
self,
batches: B,
) -> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>
pub fn batch_sampler<B>( self, batches: B, ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>
Replaces automatic batching with explicit reusable index batches.
Source§impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointDisabled>
impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointDisabled>
Sourcepub fn dataset_identity(
self,
identity: String,
) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh>
pub fn dataset_identity( self, identity: String, ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh>
Enables exact checkpointing with a caller-defined dataset identity.
The identity must describe the exact dataset contents, not merely its
Rust type or path. Empty identities are rejected by Self::build.
Wrap an immutable crate::ReplaySafeDataset in crate::ReplaySafeMap,
or use crate::TensorDataset::into_replay_safe for isolated serial tensor
inputs. Built-in samplers, identity transforms and collators support state
capture; custom components must provide their checkpoint contracts.
crate::LoaderState shows the complete save/restore flow.
Source§impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh>
impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh>
Sourcepub fn resume_from<S>(
self,
state: S,
) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointResume<S>>
pub fn resume_from<S>( self, state: S, ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointResume<S>>
Selects a saved map-loader boundary for validated resume.
Recreate the dataset and the same loader options, set its identity, then
pass a deserialized crate::LoaderState here. build() rejects
incompatible schema, identity, configuration or component state before
applying a restore. The first iter() continues the saved cursor;
later calls begin fresh passes. See crate::LoaderState for an example.
Source§impl<D, P, C, F, I, X> DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>
impl<D, P, C, F, I, X> DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>
Sourcepub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>>
pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>>
Validates configuration and builds an item-bounded, unpinned loader.
Source§impl<D, P, C, F, I, X> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>
impl<D, P, C, F, I, X> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>
Sourcepub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>>where
D: Dataset,
F: TransformFactory<D::Sample>,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: MemoryFootprint,
P: LoaderPlanConfiguration,
I: WorkerInit,
pub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>>where
D: Dataset,
F: TransformFactory<D::Sample>,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: MemoryFootprint,
P: LoaderPlanConfiguration,
I: WorkerInit,
Validates configuration and builds a byte-bounded, unpinned loader.
Source§impl<D, P, C, F, I, X, Q> DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>
impl<D, P, C, F, I, X, Q> DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>
Sourcepub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
P::Batch: PinMemory,
I: WorkerInit,
pub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
P::Batch: PinMemory,
I: WorkerInit,
Validates configuration and builds an item-bounded pinned loader.
Source§impl<D, P, C, F, I, X, Q> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>
impl<D, P, C, F, I, X, Q> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>
Sourcepub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>>where
D: Dataset,
F: TransformFactory<D::Sample>,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: MemoryFootprint,
P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
P::Batch: PinMemory,
I: WorkerInit,
pub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>>where
D: Dataset,
F: TransformFactory<D::Sample>,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: MemoryFootprint,
P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
P::Batch: PinMemory,
I: WorkerInit,
Validates configuration and builds a byte-bounded pinned loader.
Source§impl<D, P, C, F, I, N> DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointFresh>where
D: DatasetCheckpoint,
F: TransformFactory<D::Sample>,
F::Transform: WorkerCheckpoint,
P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
I: WorkerInit,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, I, N> DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointFresh>where
D: DatasetCheckpoint,
F: TransformFactory<D::Sample>,
F::Transform: WorkerCheckpoint,
P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
I: WorkerInit,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>>
pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>>
Builds a fresh exact serial loader without constructing its transform.
Source§impl<D, P, C, F, I, N, DS, SS, TS, CS> DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointResume<LoaderState<DS, SS, TS, CS>>>where
D: DatasetCheckpoint<State = DS>,
F: TransformFactory<D::Sample>,
F::Transform: WorkerCheckpoint<State = TS>,
P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C, State = SS, CoordinatorState = CS>,
I: WorkerInit,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, I, N, DS, SS, TS, CS> DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointResume<LoaderState<DS, SS, TS, CS>>>where
D: DatasetCheckpoint<State = DS>,
F: TransformFactory<D::Sample>,
F::Transform: WorkerCheckpoint<State = TS>,
P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C, State = SS, CoordinatorState = CS>,
I: WorkerInit,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn build(
self,
) -> Result<OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>, CheckpointBuildError<F::Error>>
pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>, CheckpointBuildError<F::Error>>
Validates and restores an exact serial loader transactionally.
Source§impl<D, S, C, F, I, X, M, N, K> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>where
D: Dataset,
impl<D, S, C, F, I, X, M, N, K> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>where
D: Dataset,
Sourcepub fn sampler<S2>(
self,
sampler: S2,
) -> DataLoaderBuilder<D, AutoBatch<S2>, C, F, I, X, M, N, K>
pub fn sampler<S2>( self, sampler: S2, ) -> DataLoaderBuilder<D, AutoBatch<S2>, C, F, I, X, M, N, K>
Replaces the current sampler.
Sourcepub fn shuffle(
self,
seed: u64,
) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>>
pub fn shuffle( self, seed: u64, ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>>
Replaces the current sampler with a seeded random sampler.
§Errors
Returns an invalid-configuration error for an empty dataset or an unallocatable permutation.
Sourcepub fn without_batching(
self,
) -> DataLoaderBuilder<D, NoBatch<S, DefaultConverter>, C, F, I, X, M, N, K>
pub fn without_batching( self, ) -> DataLoaderBuilder<D, NoBatch<S, DefaultConverter>, C, F, I, X, M, N, K>
Disables automatic batching and selects default conversion.
Sourcepub fn collate<C2>(
self,
collator: C2,
) -> DataLoaderBuilder<D, AutoBatch<S>, C2, F, I, X, M, N, K>
pub fn collate<C2>( self, collator: C2, ) -> DataLoaderBuilder<D, AutoBatch<S>, C2, F, I, X, M, N, K>
Replaces the automatic-batch collator.
Source§impl<D, B, C, F, I, X, M, N, K> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>where
D: Dataset,
impl<D, B, C, F, I, X, M, N, K> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>where
D: Dataset,
Sourcepub fn sampler<S>(
self,
sampler: S,
) -> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>
pub fn sampler<S>( self, sampler: S, ) -> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>
Selects a sampler; build rejects its conflict with the earlier batch sampler.
Sourcepub fn shuffle(
self,
seed: u64,
) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>>
pub fn shuffle( self, seed: u64, ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>>
Selects shuffle; build rejects its conflict with the earlier batch sampler.
§Errors
Returns an invalid-configuration error for an empty dataset or an unallocatable permutation.
Sourcepub fn without_batching(
self,
) -> DataLoaderBuilder<D, NoBatch<SequentialSampler, DefaultConverter>, C, F, I, X, M, N, K>
pub fn without_batching( self, ) -> DataLoaderBuilder<D, NoBatch<SequentialSampler, DefaultConverter>, C, F, I, X, M, N, K>
Disables batching; build rejects its conflict with the earlier batch sampler.
Sourcepub fn collate<C2>(
self,
collator: C2,
) -> DataLoaderBuilder<D, ExplicitBatches<B>, C2, F, I, X, M, N, K>
pub fn collate<C2>( self, collator: C2, ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C2, F, I, X, M, N, K>
Replaces the explicit-batch collator.
Source§impl<D, S, V, C, F, I, X, M, N, K> DataLoaderBuilder<D, NoBatch<S, V>, C, F, I, X, M, N, K>where
D: Dataset,
impl<D, S, V, C, F, I, X, M, N, K> DataLoaderBuilder<D, NoBatch<S, V>, C, F, I, X, M, N, K>where
D: Dataset,
Sourcepub fn sampler<S2>(
self,
sampler: S2,
) -> DataLoaderBuilder<D, NoBatch<S2, V>, C, F, I, X, M, N, K>
pub fn sampler<S2>( self, sampler: S2, ) -> DataLoaderBuilder<D, NoBatch<S2, V>, C, F, I, X, M, N, K>
Replaces the no-batching sampler.
Sourcepub fn shuffle(
self,
seed: u64,
) -> Result<DataLoaderBuilder<D, NoBatch<RandomSampler, V>, C, F, I, X, M, N, K>>
pub fn shuffle( self, seed: u64, ) -> Result<DataLoaderBuilder<D, NoBatch<RandomSampler, V>, C, F, I, X, M, N, K>>
Replaces the no-batching sampler with a seeded random sampler.
§Errors
Returns an invalid-configuration error for an empty dataset or an unallocatable permutation.
Sourcepub fn convert<V2>(
self,
converter: V2,
) -> DataLoaderBuilder<D, NoBatch<S, V2>, C, F, I, X, M, N, K>
pub fn convert<V2>( self, converter: V2, ) -> DataLoaderBuilder<D, NoBatch<S, V2>, C, F, I, X, M, N, K>
Replaces the no-batching converter.