Skip to main content

rusttorch_data/
stream_checkpoint.rs

1//! Save and restore stream positions at successful consumer-visible batch boundaries.
2
3use super::*;
4use crate::{
5    CheckpointDisabled, CheckpointFresh, CheckpointResume, CheckpointSourceFactory, Checkpointable,
6    CheckpointableSource, StreamCheckpointBuildError, StreamCheckpointConfiguration,
7    StreamLaneState, StreamLoaderState, TASK_RNG_DERIVATION_VERSION,
8    WORKER_SEED_DERIVATION_VERSION, WorkerCheckpoint,
9};
10use std::collections::VecDeque;
11use std::sync::atomic::{AtomicU64, Ordering};
12
13type SourceState<S> = <<S as WorkerSourceFactory>::Source as CheckpointableSource>::State;
14type TransformState<S, F> = <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as WorkerCheckpoint>::State;
15type LaneState<S, F> = StreamLaneState<SourceState<S>, TransformState<S, F>>;
16type State<S, F, C> =
17    StreamLoaderState<SourceState<S>, TransformState<S, F>, <C as Checkpointable>::State>;
18type BuildError<S, F> = StreamCheckpointBuildError<
19    <S as WorkerSourceFactory>::Error,
20    <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
21>;
22type IterError<S, F, C> = StreamLoaderError<S, C, F, NoWorkerInit>;
23
24fn invalid(reason: impl Into<String>) -> RustTorchError {
25    invalid_configuration("stream_checkpoint", reason)
26}
27
28impl<S, C, F, I, M, N> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointDisabled> {
29    /// Enables exact checkpoints for an explicit replay-safe source identity.
30    ///
31    /// The identity must change when the source contents change. Implement
32    /// [`CheckpointSourceFactory`] and [`CheckpointableSource`] for the factory
33    /// and reader; [`ExactStreamDataLoader`] provides an executable cursor and
34    /// JSON restore example. Capture on the iterator after using a batch, then
35    /// recreate the same builder and call [`Self::resume`] to continue.
36    ///
37    /// Building requires a checkpoint factory, checkpointable sources/transforms,
38    /// ordered delivery, no byte budget, no custom initializer, no persistence,
39    /// and no timeout. Ordinary iterator batching has no checkpoint method.
40    ///
41    /// ```compile_fail
42    /// use rusttorch_data::batches;
43    /// let mut iter = batches((0..4).map(Ok::<_, std::convert::Infallible>), 2, false).unwrap();
44    /// iter.checkpoint().unwrap();
45    /// ```
46    pub fn checkpointable(
47        self,
48        identity: impl Into<String>,
49    ) -> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointFresh> {
50        StreamDataLoaderBuilder {
51            factory: self.factory,
52            collator: self.collator,
53            transform_factory: self.transform_factory,
54            worker_init: self.worker_init,
55            configuration: self.configuration,
56            states: PhantomData,
57            checkpoint: CheckpointFresh {
58                identity: identity.into(),
59            },
60        }
61    }
62
63    /// Restores the input position saved after a successful batch.
64    ///
65    /// Pass the deserialized [`StreamLoaderState`] and the same content identity
66    /// used when enabling checkpointing. Builder settings, source kind and state
67    /// must match; `build()` validates the envelope and components before applying
68    /// any restore. The first `iter()` resumes that cursor, while later calls
69    /// start fresh passes. See [`ExactStreamDataLoader`] for a complete example.
70    #[allow(clippy::type_complexity)]
71    pub fn resume<SS, TS, CS>(
72        self,
73        identity: impl Into<String>,
74        state: StreamLoaderState<SS, TS, CS>,
75    ) -> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointResume<StreamLoaderState<SS, TS, CS>>>
76    {
77        StreamDataLoaderBuilder {
78            factory: self.factory,
79            collator: self.collator,
80            transform_factory: self.transform_factory,
81            worker_init: self.worker_init,
82            configuration: self.configuration,
83            states: PhantomData,
84            checkpoint: CheckpointResume {
85                identity: identity.into(),
86                state,
87            },
88        }
89    }
90}
91
92fn configuration(
93    config: StreamConfiguration,
94    pin: PinMemoryStatus,
95) -> StreamCheckpointConfiguration {
96    StreamCheckpointConfiguration {
97        workers: config.workers,
98        prefetch_factor: config.prefetch_factor,
99        batch_size: config.batch_size,
100        drop_last: config.drop_last,
101        loader_seed: config.loader_seed,
102        rank: config.rank,
103        pin_request: match config.pin_memory {
104            StreamPinRequest::Disabled => crate::CheckpointPinRequest::Disabled,
105            StreamPinRequest::Auto => crate::CheckpointPinRequest::Auto,
106            StreamPinRequest::Explicit(Device::Cuda(index)) => {
107                crate::CheckpointPinRequest::ExplicitCuda { index }
108            }
109            _ => unreachable!("pin request validated"),
110        },
111        pin_status: match pin {
112            PinMemoryStatus::Disabled => crate::CheckpointPinStatus::Disabled,
113            PinMemoryStatus::DisabledNoAccelerator => {
114                crate::CheckpointPinStatus::DisabledNoAccelerator
115            }
116            PinMemoryStatus::Enabled(Device::Cuda(index)) => {
117                crate::CheckpointPinStatus::Cuda { index }
118            }
119            _ => unreachable!("pin status validated"),
120        },
121    }
122}
123
124enum Command {
125    Apply,
126    Begin(WorkerRunContext),
127    Rollback { generation: u64, boundary: u64 },
128}
129
130enum Event<T, SE, TE> {
131    Record(WorkerRecord<T>),
132    End,
133    Source {
134        sequence: u64,
135        error: SE,
136    },
137    Transform {
138        sequence: u64,
139        logical_id: u64,
140        error: TE,
141    },
142    Protocol {
143        sequence: Option<u64>,
144        reason: String,
145    },
146    Panic,
147}
148
149impl<T, SE, TE> Event<T, SE, TE> {
150    fn sequence(&self) -> Option<u64> {
151        match self {
152            Self::Record(record) => record.sequence.map(SequenceId::get),
153            Self::Source { sequence, .. } | Self::Transform { sequence, .. } => Some(*sequence),
154            _ => None,
155        }
156    }
157}
158
159type Buffered<S, F> = (
160    usize,
161    Event<TransformOutput<S, F>, <S as WorkerSourceFactory>::Error, TransformFailure<S, F>>,
162);
163
164enum AttemptOutcome {
165    InFlight,
166    Record,
167    End,
168    SourceError,
169    TransformError,
170    Protocol,
171}
172
173struct Attempt {
174    sequence: Option<u64>,
175    _logical_id: Option<u64>,
176    _outcome: AttemptOutcome,
177}
178
179struct Completion<T, SE, TE> {
180    worker: usize,
181    generation: u64,
182    event: Event<T, SE, TE>,
183}
184type CompletionFor<S, F> =
185    Completion<TransformOutput<S, F>, <S as WorkerSourceFactory>::Error, TransformFailure<S, F>>;
186type Ack<S, F> = (
187    usize,
188    u64,
189    std::result::Result<LaneState<S, F>, BuildError<S, F>>,
190);
191
192struct Pool<S, F>
193where
194    S: WorkerSourceFactory,
195    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
196    F: TransformFactory<S::Sample>,
197    F::Transform: WorkerCheckpoint,
198{
199    commands: Vec<Sender<Command>>,
200    credits: Vec<Sender<()>>,
201    results: Receiver<CompletionFor<S, F>>,
202    #[allow(clippy::type_complexity)]
203    initialized: Receiver<(usize, std::result::Result<(), BuildError<S, F>>)>,
204    acknowledgments: Receiver<Ack<S, F>>,
205    handles: Vec<JoinHandle<()>>,
206    shutdown: CancellationToken,
207    run: WorkerRunContext,
208    committed: Arc<AtomicU64>,
209}
210
211impl<S, F> Drop for Pool<S, F>
212where
213    S: WorkerSourceFactory,
214    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
215    F: TransformFactory<S::Sample>,
216    F::Transform: WorkerCheckpoint,
217{
218    fn drop(&mut self) {
219        self.run.cancel();
220        self.shutdown.cancel();
221        for handle in self.handles.drain(..) {
222            let _ = handle.join();
223        }
224    }
225}
226
227fn validate_capacity<S, F>(config: StreamConfiguration) -> Result<usize>
228where
229    S: WorkerSourceFactory,
230    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
231    F: TransformFactory<S::Sample>,
232    F::Transform: WorkerCheckpoint,
233{
234    if config.workers == 0 || config.prefetch_factor == 0 || config.batch_size == 0 {
235        return Err(invalid(
236            "workers, prefetch_factor, and batch_size must be positive",
237        ));
238    }
239    if !config.ordered
240        || config.persistent_workers
241        || config.timeout.is_some()
242        || config.prefetch_bytes.is_some()
243    {
244        return Err(invalid(
245            "exact streams require ordered nonpersistent workers without timeout or byte accounting",
246        ));
247    }
248    let outstanding = config
249        .workers
250        .checked_mul(config.prefetch_factor)
251        .ok_or_else(|| invalid("outstanding overflow"))?;
252    // Credits for an assembling batch are returned before that batch commits.
253    // Its pre-read states therefore remain in addition to the prefetch window.
254    let journal = config
255        .prefetch_factor
256        .checked_add(config.batch_size)
257        .and_then(|n| n.checked_add(1))
258        .ok_or_else(|| invalid("journal window overflow"))?;
259    let extra = config
260        .workers
261        .checked_mul(journal)
262        .and_then(|n| n.checked_mul(size_of::<(Attempt, LaneState<S, F>)>()))
263        .and_then(|n| {
264            n.checked_add(
265                config.workers.checked_mul(
266                    size_of::<LaneState<S, F>>()
267                        .checked_mul(8)?
268                        .checked_add(8192)?
269                        .checked_add(size_of::<CapacitySlot<Ack<S, F>>>())?
270                        .checked_add(size_of::<
271                            CapacitySlot<(usize, std::result::Result<(), BuildError<S, F>>)>,
272                        >())?,
273                )?,
274            )
275        })
276        .and_then(|n| {
277            n.checked_add(outstanding.checked_mul(size_of::<CapacitySlot<CompletionFor<S, F>>>())?)
278        })
279        .and_then(|n| n.checked_add(outstanding.checked_mul(size_of::<Buffered<S, F>>())?))
280        .ok_or_else(|| invalid("snapshot storage overflow"))?;
281    validate_stream_capacity_extra::<S, F, NoWorkerInit, MemoryDisabled>(
282        config.workers,
283        config.prefetch_factor,
284        outstanding,
285        config.batch_size,
286        outstanding,
287        extra,
288    )?;
289    preflight_vec::<(Attempt, LaneState<S, F>)>(journal, "stream checkpoint journal")?;
290    Ok(journal)
291}
292
293impl<S, F> Pool<S, F>
294where
295    S: WorkerSourceFactory,
296    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
297    SourceState<S>: Send + 'static,
298    F: TransformFactory<S::Sample> + Send + Sync + 'static,
299    F::Transform: WorkerCheckpoint + Send + 'static,
300    TransformState<S, F>: Send + 'static,
301    TransformOutput<S, F>: Send + 'static,
302    TransformFailure<S, F>: Send + 'static,
303    F::Error: Send + 'static,
304{
305    fn new(
306        factory: Arc<S>,
307        transforms: Arc<F>,
308        config: StreamConfiguration,
309        source_generation: u64,
310        run: WorkerRunContext,
311        boundary: u64,
312        states: Option<Vec<LaneState<S, F>>>,
313    ) -> std::result::Result<Self, BuildError<S, F>> {
314        let journal_capacity = validate_capacity::<S, F>(config)?;
315        let (result_tx, results) = bounded_checked(
316            config.workers * config.prefetch_factor,
317            "exact stream results",
318        )?;
319        let (init_tx, initialized) =
320            bounded_checked(config.workers, "exact stream initialization")?;
321        let (ack_tx, acknowledgments) =
322            bounded_checked(config.workers, "exact stream acknowledgments")?;
323        let mut pool = Self {
324            commands: Vec::new(),
325            credits: Vec::new(),
326            results,
327            initialized,
328            acknowledgments,
329            handles: Vec::new(),
330            shutdown: CancellationToken::new(),
331            run,
332            committed: Arc::new(AtomicU64::new(boundary)),
333        };
334        let mut pending = states.map(Vec::into_iter);
335        for id in 0..config.workers {
336            let (command_tx, commands) = bounded(1);
337            let (credit_tx, credits) = bounded(config.prefetch_factor);
338            for _ in 0..config.prefetch_factor {
339                credit_tx.try_send(()).expect("fresh bounded credits");
340            }
341            pool.commands.push(command_tx);
342            pool.credits.push(credit_tx);
343            let source_factory = Arc::clone(&factory);
344            let transform_factory = Arc::clone(&transforms);
345            let run = pool.run.clone();
346            let shutdown = pool.shutdown.clone();
347            let committed = Arc::clone(&pool.committed);
348            let initialized = init_tx.clone();
349            let acknowledgments = ack_tx.clone();
350            let results = result_tx.clone();
351            let state = pending.as_mut().and_then(Iterator::next);
352            let info = WorkerInfo::from_loader_seed(
353                id,
354                config.workers,
355                config.loader_seed,
356                config.rank,
357                source_generation,
358            )?;
359            let mut journal = VecDeque::new();
360            journal
361                .try_reserve_exact(journal_capacity)
362                .map_err(|e| invalid(format!("snapshot journal unavailable: {e}")))?;
363            if journal.capacity() > journal_capacity {
364                return Err(
365                    invalid("snapshot journal exceeds validated retention capacity").into(),
366                );
367            }
368            let handle = thread::Builder::new().name(format!("rusttorch-exact-stream-{id}")).spawn(move || {
369                let mut did_initialize = false;
370                let mut generation = run.generation;
371                let outcome = catch_unwind(AssertUnwindSafe(|| with_worker_info(info, || {
372                    let context = WorkerContext::new(info, run.cancellation.clone(), run.deadline.clone());
373                    let mut source = source_factory.create(context.clone()).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })?;
374                    let mut transform = transform_factory.create(Some(&context)).map_err(|source| StreamCheckpointBuildError::TransformFactory { worker: id, source })?;
375                    if let Some(state) = &state {
376                        source.validate_snapshot(&state.source).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })?;
377                        transform.validate_snapshot(&state.transform).map_err(|source| StreamCheckpointBuildError::TransformState { worker: id, source })?;
378                    }
379                    if initialized.send((id, Ok(()))).is_err() { return Ok(()); }
380                    did_initialize = true;
381                    let mut pending = state;
382                    loop {
383                        let command = select! {
384                            recv(commands) -> command => match command { Ok(command) => command, Err(_) => return Ok(()) },
385                            recv(shutdown.signal()) -> _ => return Ok(()),
386                        };
387                        match command {
388                            Command::Apply => {
389                                if let Some(state) = pending.take() {
390                                    source.restore_validated(&state.source);
391                                    transform.restore_validated(&state.transform);
392                                }
393                            }
394                            Command::Begin(run) => {
395                                generation = run.generation;
396                                source.set_run_context(WorkerContext::new(info, run.cancellation.clone(), run.deadline.clone()));
397                                let mut previous = None;
398                                loop {
399                                    let acquired = select! {
400                                        recv(credits) -> result => result.is_ok(),
401                                        recv(run.cancellation.signal()) -> _ => false,
402                                        recv(shutdown.signal()) -> _ => false,
403                                    };
404                                    if !acquired || run.cancellation.is_cancelled() || shutdown.is_cancelled() { break; }
405                                    let boundary = committed.load(Ordering::Acquire);
406                                    while journal.front().is_some_and(|(attempt, _): &(Attempt, LaneState<S, F>)| attempt.sequence.is_some_and(|sequence| sequence < boundary)) { journal.pop_front(); }
407                                    assert!(journal.len() < journal_capacity, "validated stream snapshot window exhausted");
408                                    journal.push_back((Attempt { sequence: None, _logical_id: None, _outcome: AttemptOutcome::InFlight }, StreamLaneState { id, source: source.snapshot(), transform: transform.snapshot() }));
409                                    let event = match source.next() {
410                                        None => Event::End,
411                                        Some(Err(error)) => {
412                                            let sequence = source.error_sequence(&error).get();
413                                            if sequence < boundary || previous.is_some_and(|previous| sequence <= previous) {
414                                                Event::Protocol { sequence: Some(sequence), reason: "exact stream source error has a past or non-increasing sequence".to_owned() }
415                                            } else { Event::Source { sequence, error } }
416                                        },
417                                        Some(Ok(record)) => {
418                                            let sequence = record.sequence.map(SequenceId::get);
419                                            match sequence {
420                                                None => Event::Protocol { sequence, reason: "exact stream record is missing its sequence ID".to_owned() },
421                                                Some(sequence) if sequence < boundary || previous.is_some_and(|previous| sequence <= previous) => Event::Protocol { sequence: Some(sequence), reason: "exact stream shard emitted a past or non-increasing sequence".to_owned() },
422                                                Some(sequence) => {
423                                                    previous = Some(sequence);
424                                                    let logical_id = record.logical_id.get();
425                                                    let task = TaskContext { loader_seed: run.loader_seed, epoch: run.epoch, rank: info.rank, logical_sample: logical_id, stage: 0, cancellation: run.cancellation.clone(), deadline: run.deadline.clone() };
426                                                    match transform.transform(record.sample, &task) {
427                                                        Ok(sample) => Event::Record(WorkerRecord { sequence: record.sequence, logical_id: record.logical_id, sample }),
428                                                        Err(error) => Event::Transform { sequence, logical_id, error },
429                                                    }
430                                                }
431                                            }
432                                        }
433                                    };
434                                    let (sequence, logical_id, outcome) = match &event {
435                                        Event::Record(record) => (record.sequence.map(SequenceId::get), Some(record.logical_id.get()), AttemptOutcome::Record),
436                                        Event::End => (None, None, AttemptOutcome::End),
437                                        Event::Source { sequence, .. } => (Some(*sequence), None, AttemptOutcome::SourceError),
438                                        Event::Transform { sequence, logical_id, .. } => (Some(*sequence), Some(*logical_id), AttemptOutcome::TransformError),
439                                        Event::Protocol { sequence, .. } => (*sequence, None, AttemptOutcome::Protocol),
440                                        Event::Panic => unreachable!("panic is handled outside production"),
441                                    };
442                                    journal.back_mut().expect("pre-attempt snapshot").0 = Attempt { sequence, _logical_id: logical_id, _outcome: outcome };
443                                    let terminal = !matches!(event, Event::Record(_));
444                                    let message = Completion { worker: id, generation, event };
445                                    let sent = select! {
446                                        send(results, message) -> result => result.is_ok(),
447                                        recv(run.cancellation.signal()) -> _ => false,
448                                        recv(shutdown.signal()) -> _ => false,
449                                    };
450                                    if terminal || !sent { break; }
451                                }
452                            }
453                            Command::Rollback { generation: requested, boundary } => {
454                                let state = if requested != generation { Err(invalid("stale rollback generation").into()) } else {
455                                    let state = journal.iter().find(|(attempt, _)| attempt.sequence.is_none_or(|sequence| sequence >= boundary)).map(|(_, state)| state.clone())
456                                        .unwrap_or_else(|| StreamLaneState { id, source: source.snapshot(), transform: transform.snapshot() });
457                                    source.validate_snapshot(&state.source).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })
458                                        .and_then(|()| transform.validate_snapshot(&state.transform).map_err(|source| StreamCheckpointBuildError::TransformState { worker: id, source }))
459                                        .map(|()| { source.restore_validated(&state.source); transform.restore_validated(&state.transform); state })
460                                };
461                                journal.clear();
462                                while credits.try_recv().is_ok() {}
463                                if acknowledgments.send((id, generation, state)).is_err() { return Ok(()); }
464                            }
465                        }
466                    }
467                })));
468                match outcome {
469                    Ok(Err(error)) => { let _ = initialized.send((id, Err(error))); }
470                    Err(_) if !did_initialize => { let _ = initialized.send((id, Err(StreamCheckpointBuildError::WorkerPanic { worker: id }))); }
471                    Err(_) => {
472                        let message = Completion { worker: id, generation, event: Event::Panic };
473                        select! { send(results, message) -> _ => {}, recv(shutdown.signal()) -> _ => {} }
474                    }
475                    Ok(Ok(())) => {}
476                }
477            }).map_err(|e| invalid(format!("cannot start exact stream worker: {e}")))?;
478            pool.handles.push(handle);
479        }
480        drop(init_tx);
481        drop(ack_tx);
482        drop(result_tx);
483        let mut seen = vec![false; config.workers];
484        for _ in 0..config.workers {
485            let (id, result) = pool
486                .initialized
487                .recv()
488                .map_err(|_| invalid("worker closed during validation"))?;
489            if id >= seen.len() || std::mem::replace(&mut seen[id], true) {
490                return Err(invalid("duplicate initialization lane").into());
491            }
492            result?;
493        }
494        Ok(pool)
495    }
496
497    fn begin(&self) -> Result<()> {
498        for sender in &self.commands {
499            sender
500                .send(Command::Begin(self.run.clone()))
501                .map_err(|_| invalid("worker closed before production"))?;
502        }
503        Ok(())
504    }
505
506    fn apply(&self) -> Result<()> {
507        for sender in &self.commands {
508            sender
509                .send(Command::Apply)
510                .map_err(|_| invalid("worker closed before validated apply"))?;
511        }
512        Ok(())
513    }
514
515    fn rollback(
516        &mut self,
517        boundary: u64,
518        prefetch: usize,
519    ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
520        let generation = self.run.generation;
521        let next_generation = generation
522            .checked_add(1)
523            .filter(|g| *g < u64::MAX)
524            .ok_or_else(|| invalid("transport generation overflow"))?;
525        self.run.cancel();
526        for command in &self.commands {
527            command
528                .send(Command::Rollback {
529                    generation,
530                    boundary,
531                })
532                .map_err(|_| invalid("worker failed before rollback"))?;
533        }
534        let states = (0..self.commands.len()).map(|_| None).collect::<Vec<_>>();
535        self.collect_rollback(states, generation, next_generation, boundary, prefetch)
536    }
537
538    fn collect_rollback(
539        &mut self,
540        mut states: Vec<Option<LaneState<S, F>>>,
541        generation: u64,
542        next_generation: u64,
543        boundary: u64,
544        prefetch: usize,
545    ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
546        while states.iter().any(Option::is_none) {
547            select! {
548                recv(self.acknowledgments) -> reply => {
549                    let (id, received, state) = reply.map_err(|_| invalid("missing rollback acknowledgment"))?;
550                    if id >= states.len() || received != generation || states[id].is_some() { return Err(invalid("stale or duplicate rollback acknowledgment").into()); }
551                    states[id] = Some(state?);
552                },
553                recv(self.results) -> message => {
554                    if matches!(message, Ok(Completion { event: Event::Panic, .. }) | Err(_)) { return Err(invalid("worker failed before rollback acknowledgment").into()); }
555                },
556            }
557        }
558        self.finish_rollback(states, next_generation, boundary, prefetch)
559    }
560
561    fn finish_rollback(
562        &mut self,
563        states: Vec<Option<LaneState<S, F>>>,
564        generation: u64,
565        boundary: u64,
566        prefetch: usize,
567    ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
568        if self.acknowledgments.try_recv().is_ok() {
569            return Err(invalid("duplicate rollback acknowledgment").into());
570        }
571        while self.results.try_recv().is_ok() {}
572        for credit in &self.credits {
573            for _ in 0..prefetch {
574                credit
575                    .try_send(())
576                    .map_err(|_| invalid("rollback credits were not drained"))?;
577            }
578        }
579        self.committed.store(boundary, Ordering::Release);
580        self.run = WorkerRunContext::new(generation, self.run.loader_seed, self.run.epoch);
581        Ok(states.into_iter().map(Option::unwrap).collect())
582    }
583}
584
585/// A reusable sharded stream whose unread position can survive a restart.
586///
587/// Build it with [`StreamDataLoaderBuilder::checkpointable`]. Use this for
588/// versioned data whose source and transform state can be restored before the
589/// next unread record. Prefetched work is rolled back to the last visible batch
590/// and replayed; the checkpoint contains cursors and component state, not samples.
591///
592/// # Save a lazy shard cursor and resume the next batch
593///
594/// Each shard below saves the number of its own records already read. Validation
595/// rejects an out-of-range cursor without changing the reader. `set_run_context`
596/// replaces cancellation state after a checkpoint barrier without resetting that
597/// cursor. The example uses JSON; applications that choose this format need a
598/// `serde_json` dependency.
599///
600/// ```
601/// use std::io;
602/// use rusttorch_data::{
603///     CheckpointSourceFactory, CheckpointableSource, LogicalSampleId, SequenceId,
604///     StreamDataLoaderBuilder, StreamLoaderState, VecCollate, WorkerContext,
605///     WorkerRecord, WorkerSourceFactory,
606/// };
607///
608/// const ROWS: usize = 11;
609/// struct Rows;
610/// struct Shard { cursor: usize, context: WorkerContext }
611/// impl Shard {
612///     fn len(&self) -> usize {
613///         ROWS.saturating_sub(self.context.info.id).div_ceil(self.context.info.num_workers)
614///     }
615/// }
616/// impl Iterator for Shard {
617///     type Item = Result<WorkerRecord<usize>, io::Error>;
618///     fn next(&mut self) -> Option<Self::Item> {
619///         if self.context.cancellation.is_cancelled() || self.cursor == self.len() {
620///             return None;
621///         }
622///         let row = self.context.info.id + self.cursor * self.context.info.num_workers;
623///         self.cursor += 1;
624///         Some(Ok(WorkerRecord {
625///             sequence: Some(SequenceId::new(row as u64)),
626///             logical_id: LogicalSampleId::new(row as u64),
627///             sample: row,
628///         }))
629///     }
630/// }
631/// impl WorkerSourceFactory for Rows {
632///     type Sample = usize;
633///     type Error = io::Error;
634///     type Source = Shard;
635///     fn create(&self, context: WorkerContext) -> Result<Shard, io::Error> {
636///         Ok(Shard { cursor: 0, context })
637///     }
638///     fn exact_len(&self) -> Option<usize> { Some(ROWS) }
639/// }
640/// impl CheckpointSourceFactory for Rows {
641///     const CHECKPOINT_KIND: &'static str = "example.modulo-rows.v1";
642/// }
643/// impl CheckpointableSource for Shard {
644///     type Sample = usize;
645///     type Error = io::Error;
646///     type State = usize;
647///     fn snapshot(&self) -> usize { self.cursor }
648///     fn validate_snapshot(&self, cursor: &usize) -> Result<(), io::Error> {
649///         if *cursor > self.len() {
650///             return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid shard cursor"));
651///         }
652///         Ok(())
653///     }
654///     fn restore_validated(&mut self, cursor: &usize) { self.cursor = *cursor; }
655///     fn set_run_context(&mut self, context: WorkerContext) { self.context = context; }
656///     fn error_sequence(&self, _: &io::Error) -> SequenceId {
657///         unreachable!("this in-memory source never returns a read error")
658///     }
659/// }
660///
661/// fn main() -> Result<(), Box<dyn std::error::Error>> {
662///     let build = || StreamDataLoaderBuilder::new(Rows)
663///         .workers(3).prefetch_factor(2).batch_size(4).collate(VecCollate);
664///     let mut loader = build().checkpointable("rows-0-through-10.v1").build()?;
665///     let mut iteration = loader.iter();
666///     assert_eq!(iteration.next().unwrap()?, [0, 1, 2, 3]);
667///     let saved = serde_json::to_string(&iteration.checkpoint()?)?;
668///     let uninterrupted = iteration.collect::<Result<Vec<_>, _>>()?;
669///
670///     let state: StreamLoaderState<usize> = serde_json::from_str(&saved)?;
671///     let mut restored = build().resume("rows-0-through-10.v1", state).build()?;
672///     let remaining = restored.iter().collect::<Result<Vec<_>, _>>()?;
673///     assert_eq!(remaining, [vec![4, 5, 6, 7], vec![8, 9, 10]]);
674///     assert_eq!(remaining, uninterrupted);
675///     Ok(())
676/// }
677/// ```
678///
679/// For a file decoder, include byte offsets and all decoder state affecting future
680/// reads. Real read failures must report their reproducible global position through
681/// [`CheckpointableSource::error_sequence`]; this in-memory example has none.
682/// Use a source kind/version for the cursor format and a separate content identity
683/// for the underlying records. All shards must emit increasing sequence IDs that
684/// together cover the global sequence without gaps.
685///
686/// Exact streams require ordered delivery and checkpointable transforms/collation,
687/// with no byte budget, timeout, custom initializer or persistent workers. Save
688/// model, optimizer and training counters at the same completed batch. Application
689/// storage owns atomic writes and retention; decoding or validation errors should
690/// be propagated rather than treated as permission to restart from zero.
691pub struct ExactStreamDataLoader<S, C, F = IdentityTransformFactory, N = PinDisabled>
692where
693    S: WorkerSourceFactory,
694    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
695    F: TransformFactory<S::Sample>,
696    F::Transform: WorkerCheckpoint,
697    C: Collate<TransformOutput<S, F>> + Checkpointable,
698{
699    factory: Arc<S>,
700    transform_factory: Arc<F>,
701    collator: C,
702    configuration: StreamConfiguration,
703    identity: String,
704    pin: PinMemoryStatus,
705    length: Option<usize>,
706    pending: Option<Pool<S, F>>,
707    next_batch: u64,
708    next_sequence: u64,
709    short_tail: bool,
710    source_generation: u64,
711    next_generation: u64,
712    policies: PhantomData<N>,
713}
714
715impl<S, C, F, N> StreamDataLoaderBuilder<S, C, F, NoWorkerInit, MemoryDisabled, N, CheckpointFresh>
716where
717    S: CheckpointSourceFactory,
718    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
719    SourceState<S>: Send + 'static,
720    F: TransformFactory<S::Sample> + Send + Sync + 'static,
721    F::Transform: WorkerCheckpoint + Send + 'static,
722    TransformState<S, F>: Send + 'static,
723    TransformOutput<S, F>: Send + 'static,
724    TransformFailure<S, F>: Send + 'static,
725    F::Error: Send + 'static,
726    C: Collate<TransformOutput<S, F>> + Checkpointable,
727    N: StreamPinPolicy<S, C, F>,
728{
729    /// Validates exact capabilities and constructs suspended worker instances.
730    pub fn build(self) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>> {
731        let identity = self.checkpoint.identity.clone();
732        build(self, identity, None)
733    }
734}
735
736impl<S, C, F, N>
737    StreamDataLoaderBuilder<
738        S,
739        C,
740        F,
741        NoWorkerInit,
742        MemoryDisabled,
743        N,
744        CheckpointResume<State<S, F, C>>,
745    >
746where
747    S: CheckpointSourceFactory,
748    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
749    SourceState<S>: Send + 'static,
750    F: TransformFactory<S::Sample> + Send + Sync + 'static,
751    F::Transform: WorkerCheckpoint + Send + 'static,
752    TransformState<S, F>: Send + 'static,
753    TransformOutput<S, F>: Send + 'static,
754    TransformFailure<S, F>: Send + 'static,
755    F::Error: Send + 'static,
756    C: Collate<TransformOutput<S, F>> + Checkpointable,
757    N: StreamPinPolicy<S, C, F>,
758{
759    /// Validates the envelope before callbacks, then all components before apply.
760    pub fn build(self) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>> {
761        let StreamDataLoaderBuilder {
762            factory,
763            collator,
764            transform_factory,
765            worker_init,
766            configuration,
767            checkpoint,
768            ..
769        } = self;
770        let builder = StreamDataLoaderBuilder {
771            factory,
772            collator,
773            transform_factory,
774            worker_init,
775            configuration,
776            checkpoint: CheckpointDisabled,
777            states: PhantomData,
778        };
779        build(builder, checkpoint.identity, Some(checkpoint.state))
780    }
781}
782
783fn build<S, C, F, N, K>(
784    builder: StreamDataLoaderBuilder<S, C, F, NoWorkerInit, MemoryDisabled, N, K>,
785    identity: String,
786    state: Option<State<S, F, C>>,
787) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>>
788where
789    S: CheckpointSourceFactory,
790    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
791    SourceState<S>: Send + 'static,
792    F: TransformFactory<S::Sample> + Send + Sync + 'static,
793    F::Transform: WorkerCheckpoint + Send + 'static,
794    TransformState<S, F>: Send + 'static,
795    TransformOutput<S, F>: Send + 'static,
796    TransformFailure<S, F>: Send + 'static,
797    F::Error: Send + 'static,
798    C: Collate<TransformOutput<S, F>> + Checkpointable,
799    N: StreamPinPolicy<S, C, F>,
800{
801    let config = builder.configuration;
802    validate_capacity::<S, F>(config)?;
803    if identity.is_empty() || S::CHECKPOINT_KIND.is_empty() {
804        return Err(invalid("source identity and factory kind must not be empty").into());
805    }
806    let pin = resolve_stream_pin(config.pin_memory)?;
807    if let Some(state) = &state {
808        let batch_size = u64::try_from(config.batch_size)
809            .map_err(|_| invalid("batch size exceeds sequence range"))?;
810        let full = state
811            .next_batch
812            .checked_mul(batch_size)
813            .ok_or_else(|| invalid("batch cursor overflow"))?;
814        let short_tail = !config.drop_last
815            && state.next_batch > 0
816            && state.next_sequence < full
817            && state.next_sequence > full - batch_size;
818        if state.schema_version != 1
819            || state.source_identity != identity
820            || state.factory_kind != S::CHECKPOINT_KIND
821            || state.epoch != config.epoch
822            || state.source_generation > state.transport_generation
823            || state.transport_generation >= u64::MAX - 1
824            || state.rng_derivation_version != TASK_RNG_DERIVATION_VERSION
825            || state.worker_seed_derivation_version != WORKER_SEED_DERIVATION_VERSION
826            || state.configuration != configuration(config, pin)
827            || state.lanes.len() != config.workers
828            || state
829                .lanes
830                .iter()
831                .enumerate()
832                .any(|(id, lane)| lane.id != id)
833            || state.short_tail != short_tail
834            || (state.next_sequence != full && !short_tail)
835            || state.source_length.is_some_and(|length| {
836                state.next_sequence > length as u64
837                    || (short_tail && state.next_sequence != length as u64)
838            })
839        {
840            return Err(invalid("incompatible or corrupt stream checkpoint envelope").into());
841        }
842    }
843    let length = builder.factory.exact_len();
844    if state
845        .as_ref()
846        .is_some_and(|state| state.source_length != length)
847    {
848        return Err(invalid("source length changed").into());
849    }
850    let mut loader = ExactStreamDataLoader {
851        factory: Arc::new(builder.factory),
852        transform_factory: Arc::new(builder.transform_factory),
853        collator: builder.collator,
854        configuration: config,
855        identity,
856        pin,
857        length,
858        pending: None,
859        next_batch: state.as_ref().map_or(0, |s| s.next_batch),
860        next_sequence: state.as_ref().map_or(0, |s| s.next_sequence),
861        short_tail: state.as_ref().is_some_and(|s| s.short_tail),
862        source_generation: state.as_ref().map_or(0, |s| s.source_generation),
863        next_generation: state.as_ref().map_or(1, |s| s.transport_generation + 1),
864        policies: PhantomData,
865    };
866    let generation = state.as_ref().map_or(0, |s| s.transport_generation);
867    let pool = Pool::new(
868        Arc::clone(&loader.factory),
869        Arc::clone(&loader.transform_factory),
870        config,
871        loader.source_generation,
872        WorkerRunContext::new(generation, config.loader_seed, config.epoch),
873        loader.next_sequence,
874        state.as_ref().map(|s| s.lanes.clone()),
875    )?;
876    if let Some(state) = &state {
877        loader.collator.validate_state(&state.coordinator)?;
878        pool.apply()?;
879        loader.collator.load_validated(&state.coordinator);
880    }
881    loader.pending = Some(pool);
882    Ok(loader)
883}
884
885/// A resumable pass over an [`ExactStreamDataLoader`].
886///
887/// Consume batches like an ordinary fallible iterator. Call [`Self::checkpoint`]
888/// initially or after a successfully handled batch, then store the returned
889/// state with the matching training state. The original iterator remains usable;
890/// the owner example verifies its remaining output against a restored loader.
891pub struct ExactStreamLoaderIter<'a, S, C, F = IdentityTransformFactory, N = PinDisabled>
892where
893    S: WorkerSourceFactory,
894    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
895    F: TransformFactory<S::Sample>,
896    F::Transform: WorkerCheckpoint,
897    C: Collate<TransformOutput<S, F>> + Checkpointable,
898{
899    loader: &'a mut ExactStreamDataLoader<S, C, F, N>,
900    pool: Option<Pool<S, F>>,
901    completed: Vec<Buffered<S, F>>,
902    ended: Vec<bool>,
903    next_sequence: u64,
904    visible_sequence: u64,
905    short_tail: bool,
906    next_batch: u64,
907    stopped: bool,
908    boundary_valid: bool,
909    pending_error: Option<IterError<S, F, C>>,
910}
911
912impl<S, C, F, N> ExactStreamDataLoader<S, C, F, N>
913where
914    S: CheckpointSourceFactory,
915    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
916    SourceState<S>: Send + 'static,
917    F: TransformFactory<S::Sample> + Send + Sync + 'static,
918    F::Transform: WorkerCheckpoint + Send + 'static,
919    TransformState<S, F>: Send + 'static,
920    TransformOutput<S, F>: Send + 'static,
921    TransformFailure<S, F>: Send + 'static,
922    F::Error: Send + 'static,
923    C: Collate<TransformOutput<S, F>> + Checkpointable,
924    N: StreamPinPolicy<S, C, F>,
925{
926    /// Returns the exact global batch count, when known.
927    pub fn len(&self) -> Option<usize> {
928        self.length.map(|n| {
929            if self.configuration.drop_last {
930                n / self.configuration.batch_size
931            } else {
932                n.div_ceil(self.configuration.batch_size)
933            }
934        })
935    }
936    /// Returns whether the known batch count is zero.
937    pub fn is_empty(&self) -> Option<bool> {
938        self.len().map(|n| n == 0)
939    }
940    /// Returns the shard count.
941    pub fn workers(&self) -> usize {
942        self.configuration.workers
943    }
944    /// Returns the effective pinning policy.
945    pub fn pin_memory_status(&self) -> PinMemoryStatus {
946        self.pin
947    }
948    /// Selects a fresh epoch and discards a retained resume.
949    pub fn set_epoch(&mut self, epoch: u64) {
950        self.pending = None;
951        self.configuration.epoch = epoch;
952    }
953    /// Starts a pass, consuming a pending resume only on the first call.
954    ///
955    /// Following passes reopen the source from the beginning. Calling
956    /// [`Self::set_epoch`] before that first pass discards its saved cursor and
957    /// starts the selected epoch afresh.
958    pub fn iter(&mut self) -> ExactStreamLoaderIter<'_, S, C, F, N> {
959        let retained = self.pending.is_some();
960        let mut pending_error = None;
961        let pool = if retained {
962            self.pending.take()
963        } else {
964            let generation = self.next_generation;
965            if let Some(next) = generation.checked_add(1).filter(|g| *g < u64::MAX) {
966                self.next_generation = next;
967                self.source_generation = generation;
968                match Pool::new(
969                    Arc::clone(&self.factory),
970                    Arc::clone(&self.transform_factory),
971                    self.configuration,
972                    generation,
973                    WorkerRunContext::new(
974                        generation,
975                        self.configuration.loader_seed,
976                        self.configuration.epoch,
977                    ),
978                    0,
979                    None,
980                ) {
981                    Ok(pool) => Some(pool),
982                    Err(error) => {
983                        pending_error = Some(map_build_error::<S, F, C>(error));
984                        None
985                    }
986                }
987            } else {
988                pending_error = Some(LoaderError::Configuration(invalid("generation overflow")));
989                None
990            }
991        };
992        if let Some(pool) = &pool
993            && let Err(error) = pool.begin()
994        {
995            pending_error = Some(LoaderError::Configuration(error));
996        }
997        let next_batch = if retained { self.next_batch } else { 0 };
998        let next_sequence = if retained { self.next_sequence } else { 0 };
999        let short_tail = retained && self.short_tail;
1000        let ended = vec![false; self.configuration.workers];
1001        let completed =
1002            Vec::with_capacity(self.configuration.workers * self.configuration.prefetch_factor);
1003        ExactStreamLoaderIter {
1004            loader: self,
1005            pool,
1006            completed,
1007            ended,
1008            next_batch,
1009            next_sequence,
1010            visible_sequence: next_sequence,
1011            short_tail,
1012            stopped: false,
1013            boundary_valid: pending_error.is_none(),
1014            pending_error,
1015        }
1016    }
1017}
1018
1019fn map_build_error<S, F, C>(error: BuildError<S, F>) -> IterError<S, F, C>
1020where
1021    S: WorkerSourceFactory,
1022    F: TransformFactory<S::Sample>,
1023    C: Collate<TransformOutput<S, F>>,
1024{
1025    match error {
1026        StreamCheckpointBuildError::Configuration(e) => LoaderError::Configuration(e),
1027        StreamCheckpointBuildError::Source { worker, source } => LoaderError::StreamPipeline {
1028            batch: None,
1029            worker,
1030            sequence: None,
1031            logical_id: None,
1032            source: PipelineError::Source(source),
1033        },
1034        StreamCheckpointBuildError::TransformFactory { worker, source } => {
1035            LoaderError::StreamPipeline {
1036                batch: None,
1037                worker,
1038                sequence: None,
1039                logical_id: None,
1040                source: PipelineError::TransformInit(source),
1041            }
1042        }
1043        StreamCheckpointBuildError::TransformState { source, .. } => {
1044            LoaderError::Configuration(source)
1045        }
1046        StreamCheckpointBuildError::WorkerPanic { worker } => LoaderError::WorkerPanic {
1047            worker,
1048            batch: None,
1049        },
1050    }
1051}
1052
1053impl<S, C, F, N> ExactStreamLoaderIter<'_, S, C, F, N>
1054where
1055    S: CheckpointSourceFactory,
1056    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
1057    SourceState<S>: Send + 'static,
1058    F: TransformFactory<S::Sample> + Send + Sync + 'static,
1059    F::Transform: WorkerCheckpoint + Send + 'static,
1060    TransformState<S, F>: Send + 'static,
1061    TransformOutput<S, F>: Send + 'static,
1062    TransformFailure<S, F>: Send + 'static,
1063    F::Error: Send + 'static,
1064    C: Collate<TransformOutput<S, F>> + Checkpointable,
1065    N: StreamPinPolicy<S, C, F>,
1066{
1067    /// Captures the position before the next unconsumed batch.
1068    ///
1069    /// Call initially or after successfully using a visible batch; then save
1070    /// the returned [`StreamLoaderState`] with the same model/optimizer step.
1071    /// See [`ExactStreamDataLoader`] for a full JSON round-trip example.
1072    ///
1073    /// The barrier cancels active reads, restores unpublished source/transform
1074    /// progress and restarts the iterator from that boundary. The original
1075    /// iterator remains usable. Source/transform errors can be replayed from the
1076    /// last successful boundary; collation or pinning failures cannot. A request
1077    /// after observed exhaustion or a hidden dropped tail is rejected. Blocking
1078    /// callbacks must cooperate with cancellation for the barrier to finish.
1079    pub fn checkpoint(&mut self) -> std::result::Result<State<S, F, C>, IterError<S, F, C>> {
1080        if !self.boundary_valid {
1081            return Err(LoaderError::Checkpoint {
1082                reason: "no recoverable visible stream boundary".to_owned(),
1083            });
1084        }
1085        let coordinator = self.loader.collator.save_state();
1086        let pool = self.pool.as_mut().ok_or_else(|| LoaderError::Checkpoint {
1087            reason: "missing worker pool".to_owned(),
1088        })?;
1089        let lanes = match pool.rollback(
1090            self.visible_sequence,
1091            self.loader.configuration.prefetch_factor,
1092        ) {
1093            Ok(lanes) => lanes,
1094            Err(error) => {
1095                self.boundary_valid = false;
1096                self.stopped = true;
1097                self.pool = None;
1098                return Err(map_build_error::<S, F, C>(error));
1099            }
1100        };
1101        self.completed.clear();
1102        self.ended.fill(false);
1103        self.next_sequence = self.visible_sequence;
1104        self.stopped = false;
1105        let state = StreamLoaderState {
1106            schema_version: 1,
1107            source_identity: self.loader.identity.clone(),
1108            factory_kind: S::CHECKPOINT_KIND.to_owned(),
1109            epoch: self.loader.configuration.epoch,
1110            source_generation: self.loader.source_generation,
1111            transport_generation: pool.run.generation,
1112            next_batch: self.next_batch,
1113            next_sequence: self.visible_sequence,
1114            short_tail: self.short_tail,
1115            source_length: self.loader.length,
1116            rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
1117            worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
1118            configuration: configuration(self.loader.configuration, self.loader.pin),
1119            lanes,
1120            coordinator,
1121        };
1122        self.loader.next_generation = self.loader.next_generation.max(pool.run.generation + 1);
1123        pool.begin().map_err(LoaderError::Configuration)?;
1124        Ok(state)
1125    }
1126
1127    fn fail(
1128        &mut self,
1129        error: IterError<S, F, C>,
1130        recoverable: bool,
1131    ) -> Option<std::result::Result<C::Batch, IterError<S, F, C>>> {
1132        self.stopped = true;
1133        self.boundary_valid &= recoverable;
1134        if let Some(pool) = &self.pool {
1135            pool.run.cancel();
1136        }
1137        if !recoverable {
1138            self.pool = None;
1139        }
1140        Some(Err(error))
1141    }
1142}
1143
1144impl<S, C, F, N> Iterator for ExactStreamLoaderIter<'_, S, C, F, N>
1145where
1146    S: CheckpointSourceFactory,
1147    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
1148    SourceState<S>: Send + 'static,
1149    F: TransformFactory<S::Sample> + Send + Sync + 'static,
1150    F::Transform: WorkerCheckpoint + Send + 'static,
1151    TransformState<S, F>: Send + 'static,
1152    TransformOutput<S, F>: Send + 'static,
1153    TransformFailure<S, F>: Send + 'static,
1154    F::Error: Send + 'static,
1155    C: Collate<TransformOutput<S, F>> + Checkpointable,
1156    N: StreamPinPolicy<S, C, F>,
1157{
1158    type Item = std::result::Result<C::Batch, IterError<S, F, C>>;
1159    fn next(&mut self) -> Option<Self::Item> {
1160        if self.stopped {
1161            return None;
1162        }
1163        if let Some(error) = self.pending_error.take() {
1164            return self.fail(error, false);
1165        }
1166        let mut samples = Vec::with_capacity(self.loader.configuration.batch_size);
1167        loop {
1168            if let Some(index) = self
1169                .completed
1170                .iter()
1171                .position(|(_, event)| event.sequence() == Some(self.next_sequence))
1172            {
1173                let (worker, event) = self.completed.swap_remove(index);
1174                let record = match event {
1175                    Event::Record(record) => record,
1176                    Event::Source { sequence, error } => {
1177                        return self.fail(
1178                            LoaderError::StreamPipeline {
1179                                batch: Some(self.next_batch),
1180                                worker,
1181                                sequence: Some(sequence),
1182                                logical_id: None,
1183                                source: PipelineError::Source(error),
1184                            },
1185                            true,
1186                        );
1187                    }
1188                    Event::Transform {
1189                        sequence,
1190                        logical_id,
1191                        error,
1192                    } => {
1193                        return self.fail(
1194                            LoaderError::StreamPipeline {
1195                                batch: Some(self.next_batch),
1196                                worker,
1197                                sequence: Some(sequence),
1198                                logical_id: Some(logical_id),
1199                                source: PipelineError::Transform(error),
1200                            },
1201                            true,
1202                        );
1203                    }
1204                    _ => unreachable!("only sequenced outcomes enter reassembly"),
1205                };
1206                let Some(next) = self.next_sequence.checked_add(1) else {
1207                    return self.fail(
1208                        LoaderError::StreamProtocol {
1209                            sequence: Some(self.next_sequence),
1210                            reason: "sequence overflow".to_owned(),
1211                        },
1212                        false,
1213                    );
1214                };
1215                self.next_sequence = next;
1216                samples.push(record.sample);
1217                if self.pool.as_ref().expect("active pool").credits[worker]
1218                    .try_send(())
1219                    .is_err()
1220                {
1221                    return self.fail(
1222                        LoaderError::ChannelClosed {
1223                            batch: self.next_batch,
1224                        },
1225                        false,
1226                    );
1227                }
1228                if samples.len() == self.loader.configuration.batch_size {
1229                    break;
1230                }
1231                continue;
1232            }
1233            if self.ended.iter().all(|ended| *ended) {
1234                if !self.completed.is_empty()
1235                    || self
1236                        .loader
1237                        .length
1238                        .is_some_and(|length| self.next_sequence != length as u64)
1239                {
1240                    return self.fail(
1241                        LoaderError::StreamProtocol {
1242                            sequence: Some(self.next_sequence),
1243                            reason:
1244                                "stream ended with a missing global sequence or wrong exact length"
1245                                    .to_owned(),
1246                        },
1247                        false,
1248                    );
1249                }
1250                if samples.is_empty() || self.loader.configuration.drop_last {
1251                    self.stopped = true;
1252                    return None;
1253                }
1254                break;
1255            }
1256            // ponytail: bounded lane/window scan; retain lane counts if wide sharding matters.
1257            if self.ended.iter().enumerate().all(|(id, ended)| {
1258                *ended
1259                    || self
1260                        .completed
1261                        .iter()
1262                        .filter(|(worker, _)| *worker == id)
1263                        .count()
1264                        == self.loader.configuration.prefetch_factor
1265            }) {
1266                return self.fail(
1267                    LoaderError::StreamProtocol {
1268                        sequence: Some(self.next_sequence),
1269                        reason: "bounded stream window contains no expected sequence".to_owned(),
1270                    },
1271                    false,
1272                );
1273            }
1274            let pool = self.pool.as_ref().expect("active pool");
1275            let completion = match pool.results.recv() {
1276                Ok(completion) => completion,
1277                Err(_) => {
1278                    return self.fail(
1279                        LoaderError::ChannelClosed {
1280                            batch: self.next_batch,
1281                        },
1282                        false,
1283                    );
1284                }
1285            };
1286            if completion.generation != pool.run.generation {
1287                return self.fail(
1288                    LoaderError::StreamProtocol {
1289                        sequence: None,
1290                        reason: "stale transport message".to_owned(),
1291                    },
1292                    false,
1293                );
1294            }
1295            let worker = completion.worker;
1296            match completion.event {
1297                event @ (Event::Record(_) | Event::Source { .. } | Event::Transform { .. }) => {
1298                    let sequence = event.sequence();
1299                    if sequence.is_none_or(|sequence| sequence < self.next_sequence)
1300                        || self
1301                            .completed
1302                            .iter()
1303                            .any(|(_, previous)| previous.sequence() == sequence)
1304                    {
1305                        return self.fail(
1306                            LoaderError::StreamProtocol {
1307                                sequence,
1308                                reason: "missing, duplicate, or past sequence ID".to_owned(),
1309                            },
1310                            false,
1311                        );
1312                    }
1313                    if !matches!(event, Event::Record(_)) {
1314                        self.ended[worker] = true;
1315                    }
1316                    self.completed.push((worker, event));
1317                }
1318                Event::End => {
1319                    self.ended[worker] = true;
1320                }
1321                Event::Protocol { sequence, reason } => {
1322                    return self.fail(LoaderError::StreamProtocol { sequence, reason }, false);
1323                }
1324                Event::Panic => {
1325                    return self.fail(
1326                        LoaderError::StreamWorkerPanic {
1327                            batch: Some(self.next_batch),
1328                            worker,
1329                            sequence: None,
1330                            logical_id: None,
1331                        },
1332                        false,
1333                    );
1334                }
1335            }
1336        }
1337        let short_tail = samples.len() < self.loader.configuration.batch_size;
1338        let batch = match catch_unwind(AssertUnwindSafe(|| self.loader.collator.collate(samples))) {
1339            Ok(Ok(batch)) => batch,
1340            Ok(Err(source)) => {
1341                return self.fail(
1342                    LoaderError::Pipeline {
1343                        batch: Some(self.next_batch),
1344                        worker: None,
1345                        source: PipelineError::Collate(source),
1346                    },
1347                    false,
1348                );
1349            }
1350            Err(_) => {
1351                return self.fail(
1352                    LoaderError::CoordinatorPanic {
1353                        stage: "collation",
1354                        batch: Some(self.next_batch),
1355                    },
1356                    false,
1357                );
1358            }
1359        };
1360        let batch = if let PinMemoryStatus::Enabled(device) = self.loader.pin {
1361            match catch_unwind(AssertUnwindSafe(|| N::pin(batch, device))) {
1362                Ok(Ok(batch)) => batch,
1363                Ok(Err(source)) => {
1364                    return self.fail(
1365                        LoaderError::PinMemory {
1366                            batch: self.next_batch,
1367                            source,
1368                        },
1369                        false,
1370                    );
1371                }
1372                Err(_) => {
1373                    return self.fail(
1374                        LoaderError::CoordinatorPanic {
1375                            stage: "pinning",
1376                            batch: Some(self.next_batch),
1377                        },
1378                        false,
1379                    );
1380                }
1381            }
1382        } else {
1383            batch
1384        };
1385        let Some(next_batch) = self.next_batch.checked_add(1) else {
1386            return self.fail(
1387                LoaderError::Configuration(invalid("batch cursor overflow")),
1388                false,
1389            );
1390        };
1391        self.next_batch = next_batch;
1392        self.visible_sequence = self.next_sequence;
1393        self.short_tail = short_tail;
1394        self.pool
1395            .as_ref()
1396            .expect("active pool")
1397            .committed
1398            .store(self.visible_sequence, Ordering::Release);
1399        Some(Ok(batch))
1400    }
1401}