Skip to main content

OwnedDataLoader

Struct OwnedDataLoader 

Source
pub struct OwnedDataLoader<D, P, C, F = IdentityTransformFactory, I = NoWorkerInit, X = SerialExecution, M = MemoryDisabled, N = PinDisabled, K = CheckpointDisabled>
where D: Dataset, F: TransformFactory<D::Sample>, I: WorkerInit, M: MemoryPolicy,
{ /* private fields */ }
Expand description

An owned, re-iterable map-style data loader.

The default SerialExecution capability accepts local non-Send data. Calling DataLoaderBuilder::workers selects WorkerExecution, where positive counts share the dataset through Arc. Each worker owns a bounded task lane and one transform instance; the coordinator alone owns ordering and collation. This differs from PyTorch’s process-local dataset copies and worker-side collation while preserving bounded prefetch, deterministic routing, task-local randomness, and ordered delivery.

Implementations§

Source§

impl<D, P, C, F, N> OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>
where D: WorkerReplayDataset + Send + Sync + 'static, D::Sample: Send + 'static, D::Error: Send + 'static, F: TransformFactory<D::Sample> + Send + Sync + 'static, F::Transform: WorkerCheckpoint + Send + 'static, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static, F::Error: Send + 'static, P: CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>, N: MapPinPolicy<D, P, C, F>,

Source

pub fn iter( &mut self, ) -> WorkerLoaderIter<'_, D, P, C, F, NoWorkerInit, MemoryDisabled, N, WorkerCheckpointIteration<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State, LoaderError<PipelineError<<D as Dataset>::Error, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error, <P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Error, <F as TransformFactory<<D as Dataset>::Sample>>::Error, <NoWorkerInit as WorkerInit>::Error>>>> ⓘ

Starts the retained resume once, then fresh deterministic worker generations.

Source

pub fn set_epoch(&mut self, epoch: u64)

Discards any retained resume and selects the epoch for fresh iteration.

Source§

impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, M, N, CheckpointDisabled>
where D: Dataset, F: TransformFactory<D::Sample>, P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>, I: WorkerInit, M: MemoryPolicy, N: MapPinPolicy<D, P, C, F>,

Source

pub fn iter( &mut self, ) -> LoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ

Starts a fresh finite iteration for the configured epoch.

Source§

impl<D, P, C, F, I, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>
where D: DatasetCheckpoint, F: TransformFactory<D::Sample>, F::Transform: WorkerCheckpoint, P: LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>, I: WorkerInit, N: MapPinPolicy<D, P, C, F>,

Source

pub fn iter( &mut self, ) -> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration> ⓘ

Starts the retained resumed cursor once, then fresh exact iterations.

Source

pub fn set_epoch(&mut self, epoch: u64)

Selects the epoch used by future iterations.

Calling this before the retained resumed cursor is consumed deliberately discards that cursor and its restored transform, so the selected epoch starts as an ordinary fresh iteration instead of mixing two epochs.

Source§

impl<D, P, C, F, I, X, M, N> OwnedDataLoader<D, P, C, F, I, X, M, N, CheckpointDisabled>
where D: Dataset, F: TransformFactory<D::Sample>, P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>, I: WorkerInit, M: MemoryPolicy,

Source

pub fn set_epoch(&mut self, epoch: u64)

Selects the epoch used by future iterations.

Source§

impl<D, P, C, F, I, X, M, N, K> OwnedDataLoader<D, P, C, F, I, X, M, N, K>
where D: Dataset, F: TransformFactory<D::Sample>, P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>, I: WorkerInit, M: MemoryPolicy,

Source

pub fn len(&self) -> Option<usize>

Returns the exact number of yielded items when the source is sized.

Source

pub fn is_empty(&self) -> Option<bool>

Returns whether iteration is empty when the source is sized.

Source

pub fn epoch(&self) -> u64

Returns the current epoch.

Source

pub fn workers(&self) -> usize

Returns the configured worker count.

Source

pub fn effective_prefetch_factor(&self) -> Option<NonZeroUsize>

Returns the effective batches-prefetched-per-worker value.

Source

pub fn is_ordered(&self) -> bool

Returns whether results are configured for sampler order.

Source

pub fn pin_memory_enabled(&self) -> bool

Returns whether pinning was requested.

Source

pub fn pin_memory_status(&self) -> PinMemoryStatus

Returns the effective recursive pinning behavior.

Source

pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>

Returns the effective post-transform prefetch byte budget.

Source

pub fn timeout(&self) -> Option<Duration>

Returns the active timeout, if any.

Source

pub fn persistent_workers_enabled(&self) -> bool

Returns whether persistent workers were requested.

Source§

impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, WorkerExecution, M, N, CheckpointDisabled>
where D: Dataset + Send + Sync + 'static, D::Sample: Send + 'static, D::Error: Send + 'static, F: TransformFactory<D::Sample> + Send + Sync + 'static, F::Transform: Send + 'static, <F::Transform as Transform<D::Sample>>::Output: Send + 'static, <F::Transform as Transform<D::Sample>>::Error: Send + 'static, F::Error: Send + 'static, P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>, I: WorkerInit + Send + Sync + 'static, I::Error: Send + 'static, M: MemoryPolicy + Send + 'static, N: MapPinPolicy<D, P, C, F>,

Source

pub fn iter( &mut self, ) -> WorkerLoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ

Starts a fresh generation on a bounded worker pool, or the explicit zero-worker path.

Auto Trait Implementations§

§

impl<D, P, C, F = IdentityTransformFactory, I = NoWorkerInit, X = SerialExecution, M = MemoryDisabled, N = PinDisabled, K = CheckpointDisabled> !RefUnwindSafe for OwnedDataLoader<D, P, C, F, I, X, M, N, K>

§

impl<D, P, C, F = IdentityTransformFactory, I = NoWorkerInit, X = SerialExecution, M = MemoryDisabled, N = PinDisabled, K = CheckpointDisabled> !UnwindSafe for OwnedDataLoader<D, P, C, F, I, X, M, N, K>

§

impl<D, P, C, F, I, X, M, N, K> Freeze for OwnedDataLoader<D, P, C, F, I, X, M, N, K>
where Arc<D>: Freeze, P: Freeze, C: Freeze, Arc<F>: Freeze, Arc<I>: Freeze, Option<fn(&[<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output]) -> usize>: Freeze, Option<WorkerPool<D, F, I, M>>: Freeze, K: Freeze, PhantomData<(X, M, N)>: Freeze,

§

impl<D, P, C, F, I, X, M, N, K> Send for OwnedDataLoader<D, P, C, F, I, X, M, N, K>
where Arc<D>: Send, P: Send, C: Send, Arc<F>: Send, Arc<I>: Send, Option<fn(&[<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output]) -> usize>: Send, Option<WorkerPool<D, F, I, M>>: Send, K: Send, PhantomData<(X, M, N)>: Send,

§

impl<D, P, C, F, I, X, M, N, K> Sync for OwnedDataLoader<D, P, C, F, I, X, M, N, K>
where Arc<D>: Sync, P: Sync, C: Sync, Arc<F>: Sync, Arc<I>: Sync, Option<fn(&[<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output]) -> usize>: Sync, Option<WorkerPool<D, F, I, M>>: Sync, K: Sync, PhantomData<(X, M, N)>: Sync,

§

impl<D, P, C, F, I, X, M, N, K> Unpin for OwnedDataLoader<D, P, C, F, I, X, M, N, K>
where Arc<D>: Unpin, P: Unpin, C: Unpin, Arc<F>: Unpin, Arc<I>: Unpin, Option<fn(&[<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output]) -> usize>: Unpin, Option<WorkerPool<D, F, I, M>>: Unpin, K: Unpin, PhantomData<(X, M, N)>: Unpin,

§

impl<D, P, C, F, I, X, M, N, K> UnsafeUnpin for OwnedDataLoader<D, P, C, F, I, X, M, N, K>

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> Read<Exclusive, BecauseExclusive> for T
where T: ?Sized,

Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source§

fn vzip(self) -> V