pub struct ExactStreamDataLoader<S, C, F = IdentityTransformFactory, N = PinDisabled>where
S: WorkerSourceFactory,
S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
F: TransformFactory<S::Sample>,
F::Transform: WorkerCheckpoint,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output> + Checkpointable,{ /* private fields */ }Expand description
A reusable sharded stream whose unread position can survive a restart.
Build it with StreamDataLoaderBuilder::checkpointable. Use this for
versioned data whose source and transform state can be restored before the
next unread record. Prefetched work is rolled back to the last visible batch
and replayed; the checkpoint contains cursors and component state, not samples.
§Save a lazy shard cursor and resume the next batch
Each shard below saves the number of its own records already read. Validation
rejects an out-of-range cursor without changing the reader. set_run_context
replaces cancellation state after a checkpoint barrier without resetting that
cursor. The example uses JSON; applications that choose this format need a
serde_json dependency.
use std::io;
use rusttorch_data::{
CheckpointSourceFactory, CheckpointableSource, LogicalSampleId, SequenceId,
StreamDataLoaderBuilder, StreamLoaderState, VecCollate, WorkerContext,
WorkerRecord, WorkerSourceFactory,
};
const ROWS: usize = 11;
struct Rows;
struct Shard { cursor: usize, context: WorkerContext }
impl Shard {
fn len(&self) -> usize {
ROWS.saturating_sub(self.context.info.id).div_ceil(self.context.info.num_workers)
}
}
impl Iterator for Shard {
type Item = Result<WorkerRecord<usize>, io::Error>;
fn next(&mut self) -> Option<Self::Item> {
if self.context.cancellation.is_cancelled() || self.cursor == self.len() {
return None;
}
let row = self.context.info.id + self.cursor * self.context.info.num_workers;
self.cursor += 1;
Some(Ok(WorkerRecord {
sequence: Some(SequenceId::new(row as u64)),
logical_id: LogicalSampleId::new(row as u64),
sample: row,
}))
}
}
impl WorkerSourceFactory for Rows {
type Sample = usize;
type Error = io::Error;
type Source = Shard;
fn create(&self, context: WorkerContext) -> Result<Shard, io::Error> {
Ok(Shard { cursor: 0, context })
}
fn exact_len(&self) -> Option<usize> { Some(ROWS) }
}
impl CheckpointSourceFactory for Rows {
const CHECKPOINT_KIND: &'static str = "example.modulo-rows.v1";
}
impl CheckpointableSource for Shard {
type Sample = usize;
type Error = io::Error;
type State = usize;
fn snapshot(&self) -> usize { self.cursor }
fn validate_snapshot(&self, cursor: &usize) -> Result<(), io::Error> {
if *cursor > self.len() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid shard cursor"));
}
Ok(())
}
fn restore_validated(&mut self, cursor: &usize) { self.cursor = *cursor; }
fn set_run_context(&mut self, context: WorkerContext) { self.context = context; }
fn error_sequence(&self, _: &io::Error) -> SequenceId {
unreachable!("this in-memory source never returns a read error")
}
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let build = || StreamDataLoaderBuilder::new(Rows)
.workers(3).prefetch_factor(2).batch_size(4).collate(VecCollate);
let mut loader = build().checkpointable("rows-0-through-10.v1").build()?;
let mut iteration = loader.iter();
assert_eq!(iteration.next().unwrap()?, [0, 1, 2, 3]);
let saved = serde_json::to_string(&iteration.checkpoint()?)?;
let uninterrupted = iteration.collect::<Result<Vec<_>, _>>()?;
let state: StreamLoaderState<usize> = serde_json::from_str(&saved)?;
let mut restored = build().resume("rows-0-through-10.v1", state).build()?;
let remaining = restored.iter().collect::<Result<Vec<_>, _>>()?;
assert_eq!(remaining, [vec![4, 5, 6, 7], vec![8, 9, 10]]);
assert_eq!(remaining, uninterrupted);
Ok(())
}For a file decoder, include byte offsets and all decoder state affecting future
reads. Real read failures must report their reproducible global position through
CheckpointableSource::error_sequence; this in-memory example has none.
Use a source kind/version for the cursor format and a separate content identity
for the underlying records. All shards must emit increasing sequence IDs that
together cover the global sequence without gaps.
Exact streams require ordered delivery and checkpointable transforms/collation, with no byte budget, timeout, custom initializer or persistent workers. Save model, optimizer and training counters at the same completed batch. Application storage owns atomic writes and retention; decoding or validation errors should be propagated rather than treated as permission to restart from zero.
Implementations§
Source§impl<S, C, F, N> ExactStreamDataLoader<S, C, F, N>where
S: CheckpointSourceFactory,
S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
<<S as WorkerSourceFactory>::Source as CheckpointableSource>::State: Send + 'static,
F: TransformFactory<S::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint + Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output> + Checkpointable,
N: StreamPinPolicy<S, C, F>,
impl<S, C, F, N> ExactStreamDataLoader<S, C, F, N>where
S: CheckpointSourceFactory,
S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
<<S as WorkerSourceFactory>::Source as CheckpointableSource>::State: Send + 'static,
F: TransformFactory<S::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint + Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output> + Checkpointable,
N: StreamPinPolicy<S, C, F>,
Sourcepub fn pin_memory_status(&self) -> PinMemoryStatus
pub fn pin_memory_status(&self) -> PinMemoryStatus
Returns the effective pinning policy.
Sourcepub fn iter(&mut self) -> ExactStreamLoaderIter<'_, S, C, F, N> ⓘ
pub fn iter(&mut self) -> ExactStreamLoaderIter<'_, S, C, F, N> ⓘ
Starts a pass, consuming a pending resume only on the first call.
Following passes reopen the source from the beginning. Calling
Self::set_epoch before that first pass discards its saved cursor and
starts the selected epoch afresh.