Skip to main content

DataLoaderBuilder

Struct DataLoaderBuilder 

Source
pub struct DataLoaderBuilder<D, P, C, F = IdentityTransformFactory, I = NoWorkerInit, X = SerialExecution, M = MemoryDisabled, N = PinDisabled, K = CheckpointDisabled> { /* private fields */ }
Expand description

Builder for an owned, re-iterable map-style data loader.

Implementations§

Source§

impl<D, P, C, F, N> DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, CheckpointFresh>
where D: WorkerReplayDataset + Send + Sync + 'static, D::Sample: Send + 'static, D::Error: Send + 'static, F: TransformFactory<D::Sample> + Send + Sync + 'static, F::Transform: WorkerCheckpoint + Send + 'static, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static, F::Error: Send + 'static, P: LoaderPlanConfiguration + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>, N: MapPinPolicy<D, P, C, F>,

Source

pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>>

Builds an exact replay loader with explicit immutable dataset evidence.

Positive workers require ordered delivery, no timeout or persistence, disabled byte accounting, and no worker initializer. Snapshot heap contents are item-bounded, not measured as total resident bytes.

Source§

impl<D, P, C, F, N, SS, TS, CS> DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, CheckpointResume<LoaderState<(), SS, WorkerTransformState<TS>, CS>>>
where D: WorkerReplayDataset + Send + Sync + 'static, D::Sample: Send + 'static, D::Error: Send + 'static, F: TransformFactory<D::Sample> + Send + Sync + 'static, F::Transform: WorkerCheckpoint<State = TS> + Send + 'static, TS: Clone + Send + 'static, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static, F::Error: Send + 'static, P: LoaderPlanConfiguration + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C, State = SS, CoordinatorState = CS>, N: MapPinPolicy<D, P, C, F>,

Source

pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>, CheckpointBuildError<F::Error>>

Validates every component before restoring retained worker instances.

Source§

impl<D> DataLoaderBuilder<D, AutoBatch<SequentialSampler>, DefaultCollator>
where D: Dataset,

Source

pub fn new(dataset: D) -> Self

Creates the default serial builder for dataset.

Source§

impl<D, P, C, F, I, X, M, N, K> DataLoaderBuilder<D, P, C, F, I, X, M, N, K>

Source

pub fn batch_size(self, batch_size: usize) -> Self

Sets the automatic batch size.

Source

pub fn drop_last(self, drop_last: bool) -> Self

Selects whether a short final automatic batch is omitted.

Source

pub fn workers( self, workers: usize, ) -> DataLoaderBuilder<D, P, C, F, I, WorkerExecution, M, N, K>

Selects bounded map-worker execution and its thread-safe capability.

Positive counts share the map dataset through Arc, fetch and transform in deterministic worker lanes, and collate on the calling thread. Passing zero keeps serial execution, but selecting this method still requires worker-safe types at compile time.

Source

pub fn seed(self, seed: u64) -> Self

Sets the seed used by deterministic task and worker contexts.

Source

pub fn epoch(self, epoch: u64) -> Self

Sets the initial epoch used by sampling and task contexts.

Source

pub fn rank(self, rank: usize) -> Self

Sets the distributed rank included in deterministic contexts.

Source

pub fn transform<T>( self, transform: T, ) -> DataLoaderBuilder<D, P, C, CloneTransformFactory<T>, I, X, M, N, K>

Clones transform once for each serial iterator or worker lifecycle.

Source

pub fn transform_factory<F2>( self, factory: F2, ) -> DataLoaderBuilder<D, P, C, F2, I, X, M, N, K>

Replaces transform construction with an explicit typed factory.

Source

pub fn checkpoint_stateless( self, ) -> DataLoaderBuilder<D, P, C, StatelessTransformFactory<F>, I, X, M, N, K>

Marks the current factory’s transform as explicitly stateless for exact checkpointing.

Source

pub fn checkpoint_transactional( self, ) -> DataLoaderBuilder<D, P, C, TransactionalTransformFactory<F>, I, X, M, N, K>

Marks the current factory’s transform as transactional for exact checkpointing.

Source

pub fn worker_init<I2>( self, worker_init: I2, ) -> DataLoaderBuilder<D, P, C, F, I2, X, M, N, K>

Stores the initializer used by future positive-worker execution.

Source

pub fn persistent_workers(self, persistent: bool) -> Self

Selects whether positive-worker iterators reuse loader-owned threads.

Persistent workers keep their initial crate::WorkerInfo seeds and factory-created state across epochs. Each iterator still receives a fresh generation cancellation token and deadline, and dropping either the iterator or loader cooperatively wakes and joins the affected work.

Source

pub fn timeout(self, timeout: Duration) -> Self

Sets the cooperative timeout for each blocking Iterator::next call.

Duration::ZERO disables the timeout. Expiry cancels the current generation, drains it to quiescence, yields one typed timeout error, and then ends the iterator. User dataset and transform code must observe its crate::WorkerContext or crate::TaskContext to stop promptly; Rust threads are never force-cancelled. A nonzero timeout that exceeds the platform monotonic clock range is rejected by Self::build.

Source

pub fn ordered(self, ordered: bool) -> Self

Selects ordered or completion-order delivery.

Source

pub fn in_order(self, in_order: bool) -> Self

Selects sampler-order or completion-order delivery.

Source

pub fn pin_memory( self, ) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Auto>, K>

Enables automatic recursive pinning for CUDA device zero when available.

Enabling pinning requires the final batch type to implement PinMemory at build time:

ⓘ
use std::convert::Infallible;
use rusttorch_data::{DataLoader, Dataset, FnCollate};

struct Rows;
struct Batch;
impl Dataset for Rows {
    type Sample = u8;
    type Error = Infallible;
    fn len(&self) -> usize { 1 }
    fn get(&self, _: usize) -> Result<u8, Infallible> { Ok(1) }
}

let _ = DataLoader::builder(Rows)
    .collate(FnCollate::new(|_: Vec<u8>| Ok::<_, Infallible>(Batch)))
    .pin_memory()
    .build();
Source

pub fn pin_memory_for( self, device: Device, ) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Explicit>, K>

Enables recursive pinning for one explicit, available CUDA device.

Source

pub fn prefetch_factor(self, factor: usize) -> Self

Sets bounded batches prefetched per worker.

The worker count multiplied by this factor is checked before any worker starts and is also the global outstanding-work credit limit. Capacities that overflow crossbeam’s ring arithmetic or whose conservative aggregate of concrete task/control/completion slots, worker bookkeeping, and channel control-block allowances exceeds the checked 64 MiB queue-allocation ceiling are rejected during build.

Source

pub fn prefetch_bytes( self, limit: NonZeroUsize, ) -> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, N, K>

Enables a nonzero byte budget for final post-transform prefetched data.

The coordinator’s active collation batch is outside this queue budget and remains bounded by the configured batch size.

Enabling the budget requires the final transformed type to implement MemoryFootprint at build time:

ⓘ
use std::{convert::Infallible, num::NonZeroUsize};
use rusttorch_data::{DataLoader, Dataset, VecCollate};

struct Sample;
struct Rows;
impl Dataset for Rows {
    type Sample = Sample;
    type Error = Infallible;
    fn len(&self) -> usize { 1 }
    fn get(&self, _: usize) -> Result<Sample, Infallible> { Ok(Sample) }
}

let _ = DataLoader::builder(Rows)
    .workers(1)
    .collate(VecCollate)
    .prefetch_bytes(NonZeroUsize::MIN)
    .build();
Source

pub fn batch_sampler<B>( self, batches: B, ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>

Replaces automatic batching with explicit reusable index batches.

Source§

impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointDisabled>

Source

pub fn dataset_identity( self, identity: String, ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh>

Enables exact checkpointing with a caller-defined dataset identity.

The identity must describe the exact dataset contents, not merely its Rust type or path. Empty identities are rejected by Self::build.

Source§

impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh>

Source

pub fn resume_from<S>( self, state: S, ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointResume<S>>

Selects a typed serial loader checkpoint for validated resume.

Source§

impl<D, P, C, F, I, X> DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>

Source

pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>>
where D: Dataset, F: TransformFactory<D::Sample>, P: LoaderPlanConfiguration, I: WorkerInit,

Validates configuration and builds an item-bounded, unpinned loader.

Source§

impl<D, P, C, F, I, X> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>

Source

pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>>
where D: Dataset, F: TransformFactory<D::Sample>, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: MemoryFootprint, P: LoaderPlanConfiguration, I: WorkerInit,

Validates configuration and builds a byte-bounded, unpinned loader.

Source§

impl<D, P, C, F, I, X, Q> DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>

Source

pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>>
where D: Dataset, F: TransformFactory<D::Sample>, P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>, P::Batch: PinMemory, I: WorkerInit,

Validates configuration and builds an item-bounded pinned loader.

Source§

impl<D, P, C, F, I, X, Q> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>

Source

pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>>
where D: Dataset, F: TransformFactory<D::Sample>, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: MemoryFootprint, P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>, P::Batch: PinMemory, I: WorkerInit,

Validates configuration and builds a byte-bounded pinned loader.

Source§

impl<D, P, C, F, I, N> DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointFresh>
where D: DatasetCheckpoint, F: TransformFactory<D::Sample>, F::Transform: WorkerCheckpoint, P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>, I: WorkerInit, N: MapPinPolicy<D, P, C, F>,

Source

pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>>

Builds a fresh exact serial loader without constructing its transform.

Source§

impl<D, P, C, F, I, N, DS, SS, TS, CS> DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointResume<LoaderState<DS, SS, TS, CS>>>
where D: DatasetCheckpoint<State = DS>, F: TransformFactory<D::Sample>, F::Transform: WorkerCheckpoint<State = TS>, P: LoaderPlanConfiguration + LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C, State = SS, CoordinatorState = CS>, I: WorkerInit, N: MapPinPolicy<D, P, C, F>,

Source

pub fn build( self, ) -> Result<OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>, CheckpointBuildError<F::Error>>

Validates and restores an exact serial loader transactionally.

Source§

impl<D, S, C, F, I, X, M, N, K> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>
where D: Dataset,

Source

pub fn sampler<S2>( self, sampler: S2, ) -> DataLoaderBuilder<D, AutoBatch<S2>, C, F, I, X, M, N, K>

Replaces the current sampler.

Source

pub fn shuffle( self, seed: u64, ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>>

Replaces the current sampler with a seeded random sampler.

§Errors

Returns an invalid-configuration error for an empty dataset or an unallocatable permutation.

Source

pub fn without_batching( self, ) -> DataLoaderBuilder<D, NoBatch<S, DefaultConverter>, C, F, I, X, M, N, K>

Disables automatic batching and selects default conversion.

Source

pub fn collate<C2>( self, collator: C2, ) -> DataLoaderBuilder<D, AutoBatch<S>, C2, F, I, X, M, N, K>

Replaces the automatic-batch collator.

Source§

impl<D, B, C, F, I, X, M, N, K> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>
where D: Dataset,

Source

pub fn sampler<S>( self, sampler: S, ) -> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>

Selects a sampler; build rejects its conflict with the earlier batch sampler.

Source

pub fn shuffle( self, seed: u64, ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>>

Selects shuffle; build rejects its conflict with the earlier batch sampler.

§Errors

Returns an invalid-configuration error for an empty dataset or an unallocatable permutation.

Source

pub fn without_batching( self, ) -> DataLoaderBuilder<D, NoBatch<SequentialSampler, DefaultConverter>, C, F, I, X, M, N, K>

Disables batching; build rejects its conflict with the earlier batch sampler.

Source

pub fn collate<C2>( self, collator: C2, ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C2, F, I, X, M, N, K>

Replaces the explicit-batch collator.

Source§

impl<D, S, V, C, F, I, X, M, N, K> DataLoaderBuilder<D, NoBatch<S, V>, C, F, I, X, M, N, K>
where D: Dataset,

Source

pub fn sampler<S2>( self, sampler: S2, ) -> DataLoaderBuilder<D, NoBatch<S2, V>, C, F, I, X, M, N, K>

Replaces the no-batching sampler.

Source

pub fn shuffle( self, seed: u64, ) -> Result<DataLoaderBuilder<D, NoBatch<RandomSampler, V>, C, F, I, X, M, N, K>>

Replaces the no-batching sampler with a seeded random sampler.

§Errors

Returns an invalid-configuration error for an empty dataset or an unallocatable permutation.

Source

pub fn convert<V2>( self, converter: V2, ) -> DataLoaderBuilder<D, NoBatch<S, V2>, C, F, I, X, M, N, K>

Replaces the no-batching converter.

Auto Trait Implementations§

§

impl<D, P, C, F, I, X, M, N, K> Freeze for DataLoaderBuilder<D, P, C, F, I, X, M, N, K>

§

impl<D, P, C, F, I, X, M, N, K> RefUnwindSafe for DataLoaderBuilder<D, P, C, F, I, X, M, N, K>

§

impl<D, P, C, F, I, X, M, N, K> Send for DataLoaderBuilder<D, P, C, F, I, X, M, N, K>
where D: Send, P: Send, C: Send, F: Send, I: Send, K: Send, PhantomData<(X, M, N)>: Send,

§

impl<D, P, C, F, I, X, M, N, K> Sync for DataLoaderBuilder<D, P, C, F, I, X, M, N, K>
where D: Sync, P: Sync, C: Sync, F: Sync, I: Sync, K: Sync, PhantomData<(X, M, N)>: Sync,

§

impl<D, P, C, F, I, X, M, N, K> Unpin for DataLoaderBuilder<D, P, C, F, I, X, M, N, K>
where D: Unpin, P: Unpin, C: Unpin, F: Unpin, I: Unpin, K: Unpin, PhantomData<(X, M, N)>: Unpin,

§

impl<D, P, C, F, I, X, M, N, K> UnsafeUnpin for DataLoaderBuilder<D, P, C, F, I, X, M, N, K>

§

impl<D, P, C, F, I, X, M, N, K> UnwindSafe for DataLoaderBuilder<D, P, C, F, I, X, M, N, K>

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> Read<Exclusive, BecauseExclusive> for T
where T: ?Sized,

Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source§

fn vzip(self) -> V