Skip to main content

rusttorch_data/
stream_checkpoint.rs

1//! Retained-source lifecycle for exact streams. Ordinary streams have no journal.
2
3use super::*;
4use crate::{
5    CheckpointDisabled, CheckpointFresh, CheckpointResume, CheckpointSourceFactory, Checkpointable,
6    CheckpointableSource, StreamCheckpointBuildError, StreamCheckpointConfiguration,
7    StreamLaneState, StreamLoaderState, TASK_RNG_DERIVATION_VERSION,
8    WORKER_SEED_DERIVATION_VERSION, WorkerCheckpoint,
9};
10use std::collections::VecDeque;
11use std::sync::atomic::{AtomicU64, Ordering};
12
13type SourceState<S> = <<S as WorkerSourceFactory>::Source as CheckpointableSource>::State;
14type TransformState<S, F> = <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as WorkerCheckpoint>::State;
15type LaneState<S, F> = StreamLaneState<SourceState<S>, TransformState<S, F>>;
16type State<S, F, C> =
17    StreamLoaderState<SourceState<S>, TransformState<S, F>, <C as Checkpointable>::State>;
18type BuildError<S, F> = StreamCheckpointBuildError<
19    <S as WorkerSourceFactory>::Error,
20    <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
21>;
22type IterError<S, F, C> = StreamLoaderError<S, C, F, NoWorkerInit>;
23
24fn invalid(reason: impl Into<String>) -> RustTorchError {
25    invalid_configuration("stream_checkpoint", reason)
26}
27
28impl<S, C, F, I, M, N> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointDisabled> {
29    /// Enables exact checkpoints for an explicit replay-safe source identity.
30    ///
31    /// Building requires a checkpoint factory, checkpointable sources/transforms,
32    /// ordered delivery, no byte budget, no custom initializer, no persistence,
33    /// and no timeout. Ordinary iterator batching has no checkpoint method.
34    ///
35    /// ```compile_fail
36    /// use rusttorch_data::batches;
37    /// let mut iter = batches((0..4).map(Ok::<_, std::convert::Infallible>), 2, false).unwrap();
38    /// iter.checkpoint().unwrap();
39    /// ```
40    pub fn checkpointable(
41        self,
42        identity: impl Into<String>,
43    ) -> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointFresh> {
44        StreamDataLoaderBuilder {
45            factory: self.factory,
46            collator: self.collator,
47            transform_factory: self.transform_factory,
48            worker_init: self.worker_init,
49            configuration: self.configuration,
50            states: PhantomData,
51            checkpoint: CheckpointFresh {
52                identity: identity.into(),
53            },
54        }
55    }
56
57    /// Selects a serialized next-visible-batch stream checkpoint for resume.
58    #[allow(clippy::type_complexity)]
59    pub fn resume<SS, TS, CS>(
60        self,
61        identity: impl Into<String>,
62        state: StreamLoaderState<SS, TS, CS>,
63    ) -> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointResume<StreamLoaderState<SS, TS, CS>>>
64    {
65        StreamDataLoaderBuilder {
66            factory: self.factory,
67            collator: self.collator,
68            transform_factory: self.transform_factory,
69            worker_init: self.worker_init,
70            configuration: self.configuration,
71            states: PhantomData,
72            checkpoint: CheckpointResume {
73                identity: identity.into(),
74                state,
75            },
76        }
77    }
78}
79
80fn configuration(
81    config: StreamConfiguration,
82    pin: PinMemoryStatus,
83) -> StreamCheckpointConfiguration {
84    StreamCheckpointConfiguration {
85        workers: config.workers,
86        prefetch_factor: config.prefetch_factor,
87        batch_size: config.batch_size,
88        drop_last: config.drop_last,
89        loader_seed: config.loader_seed,
90        rank: config.rank,
91        pin_request: match config.pin_memory {
92            StreamPinRequest::Disabled => crate::CheckpointPinRequest::Disabled,
93            StreamPinRequest::Auto => crate::CheckpointPinRequest::Auto,
94            StreamPinRequest::Explicit(Device::Cuda(index)) => {
95                crate::CheckpointPinRequest::ExplicitCuda { index }
96            }
97            _ => unreachable!("pin request validated"),
98        },
99        pin_status: match pin {
100            PinMemoryStatus::Disabled => crate::CheckpointPinStatus::Disabled,
101            PinMemoryStatus::DisabledNoAccelerator => {
102                crate::CheckpointPinStatus::DisabledNoAccelerator
103            }
104            PinMemoryStatus::Enabled(Device::Cuda(index)) => {
105                crate::CheckpointPinStatus::Cuda { index }
106            }
107            _ => unreachable!("pin status validated"),
108        },
109    }
110}
111
112enum Command {
113    Apply,
114    Begin(WorkerRunContext),
115    Rollback { generation: u64, boundary: u64 },
116}
117
118enum Event<T, SE, TE> {
119    Record(WorkerRecord<T>),
120    End,
121    Source {
122        sequence: u64,
123        error: SE,
124    },
125    Transform {
126        sequence: u64,
127        logical_id: u64,
128        error: TE,
129    },
130    Protocol {
131        sequence: Option<u64>,
132        reason: String,
133    },
134    Panic,
135}
136
137impl<T, SE, TE> Event<T, SE, TE> {
138    fn sequence(&self) -> Option<u64> {
139        match self {
140            Self::Record(record) => record.sequence.map(SequenceId::get),
141            Self::Source { sequence, .. } | Self::Transform { sequence, .. } => Some(*sequence),
142            _ => None,
143        }
144    }
145}
146
147type Buffered<S, F> = (
148    usize,
149    Event<TransformOutput<S, F>, <S as WorkerSourceFactory>::Error, TransformFailure<S, F>>,
150);
151
152enum AttemptOutcome {
153    InFlight,
154    Record,
155    End,
156    SourceError,
157    TransformError,
158    Protocol,
159}
160
161struct Attempt {
162    sequence: Option<u64>,
163    _logical_id: Option<u64>,
164    _outcome: AttemptOutcome,
165}
166
167struct Completion<T, SE, TE> {
168    worker: usize,
169    generation: u64,
170    event: Event<T, SE, TE>,
171}
172type CompletionFor<S, F> =
173    Completion<TransformOutput<S, F>, <S as WorkerSourceFactory>::Error, TransformFailure<S, F>>;
174type Ack<S, F> = (
175    usize,
176    u64,
177    std::result::Result<LaneState<S, F>, BuildError<S, F>>,
178);
179
180struct Pool<S, F>
181where
182    S: WorkerSourceFactory,
183    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
184    F: TransformFactory<S::Sample>,
185    F::Transform: WorkerCheckpoint,
186{
187    commands: Vec<Sender<Command>>,
188    credits: Vec<Sender<()>>,
189    results: Receiver<CompletionFor<S, F>>,
190    #[allow(clippy::type_complexity)]
191    initialized: Receiver<(usize, std::result::Result<(), BuildError<S, F>>)>,
192    acknowledgments: Receiver<Ack<S, F>>,
193    handles: Vec<JoinHandle<()>>,
194    shutdown: CancellationToken,
195    run: WorkerRunContext,
196    committed: Arc<AtomicU64>,
197}
198
199impl<S, F> Drop for Pool<S, F>
200where
201    S: WorkerSourceFactory,
202    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
203    F: TransformFactory<S::Sample>,
204    F::Transform: WorkerCheckpoint,
205{
206    fn drop(&mut self) {
207        self.run.cancel();
208        self.shutdown.cancel();
209        for handle in self.handles.drain(..) {
210            let _ = handle.join();
211        }
212    }
213}
214
215fn validate_capacity<S, F>(config: StreamConfiguration) -> Result<usize>
216where
217    S: WorkerSourceFactory,
218    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
219    F: TransformFactory<S::Sample>,
220    F::Transform: WorkerCheckpoint,
221{
222    if config.workers == 0 || config.prefetch_factor == 0 || config.batch_size == 0 {
223        return Err(invalid(
224            "workers, prefetch_factor, and batch_size must be positive",
225        ));
226    }
227    if !config.ordered
228        || config.persistent_workers
229        || config.timeout.is_some()
230        || config.prefetch_bytes.is_some()
231    {
232        return Err(invalid(
233            "exact streams require ordered nonpersistent workers without timeout or byte accounting",
234        ));
235    }
236    let outstanding = config
237        .workers
238        .checked_mul(config.prefetch_factor)
239        .ok_or_else(|| invalid("outstanding overflow"))?;
240    // Credits for an assembling batch are returned before that batch commits.
241    // Its pre-read states therefore remain in addition to the prefetch window.
242    let journal = config
243        .prefetch_factor
244        .checked_add(config.batch_size)
245        .and_then(|n| n.checked_add(1))
246        .ok_or_else(|| invalid("journal window overflow"))?;
247    let extra = config
248        .workers
249        .checked_mul(journal)
250        .and_then(|n| n.checked_mul(size_of::<(Attempt, LaneState<S, F>)>()))
251        .and_then(|n| {
252            n.checked_add(
253                config.workers.checked_mul(
254                    size_of::<LaneState<S, F>>()
255                        .checked_mul(8)?
256                        .checked_add(8192)?
257                        .checked_add(size_of::<CapacitySlot<Ack<S, F>>>())?
258                        .checked_add(size_of::<
259                            CapacitySlot<(usize, std::result::Result<(), BuildError<S, F>>)>,
260                        >())?,
261                )?,
262            )
263        })
264        .and_then(|n| {
265            n.checked_add(outstanding.checked_mul(size_of::<CapacitySlot<CompletionFor<S, F>>>())?)
266        })
267        .and_then(|n| n.checked_add(outstanding.checked_mul(size_of::<Buffered<S, F>>())?))
268        .ok_or_else(|| invalid("snapshot storage overflow"))?;
269    validate_stream_capacity_extra::<S, F, NoWorkerInit, MemoryDisabled>(
270        config.workers,
271        config.prefetch_factor,
272        outstanding,
273        config.batch_size,
274        outstanding,
275        extra,
276    )?;
277    preflight_vec::<(Attempt, LaneState<S, F>)>(journal, "stream checkpoint journal")?;
278    Ok(journal)
279}
280
281impl<S, F> Pool<S, F>
282where
283    S: WorkerSourceFactory,
284    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
285    SourceState<S>: Send + 'static,
286    F: TransformFactory<S::Sample> + Send + Sync + 'static,
287    F::Transform: WorkerCheckpoint + Send + 'static,
288    TransformState<S, F>: Send + 'static,
289    TransformOutput<S, F>: Send + 'static,
290    TransformFailure<S, F>: Send + 'static,
291    F::Error: Send + 'static,
292{
293    fn new(
294        factory: Arc<S>,
295        transforms: Arc<F>,
296        config: StreamConfiguration,
297        source_generation: u64,
298        run: WorkerRunContext,
299        boundary: u64,
300        states: Option<Vec<LaneState<S, F>>>,
301    ) -> std::result::Result<Self, BuildError<S, F>> {
302        let journal_capacity = validate_capacity::<S, F>(config)?;
303        let (result_tx, results) = bounded_checked(
304            config.workers * config.prefetch_factor,
305            "exact stream results",
306        )?;
307        let (init_tx, initialized) =
308            bounded_checked(config.workers, "exact stream initialization")?;
309        let (ack_tx, acknowledgments) =
310            bounded_checked(config.workers, "exact stream acknowledgments")?;
311        let mut pool = Self {
312            commands: Vec::new(),
313            credits: Vec::new(),
314            results,
315            initialized,
316            acknowledgments,
317            handles: Vec::new(),
318            shutdown: CancellationToken::new(),
319            run,
320            committed: Arc::new(AtomicU64::new(boundary)),
321        };
322        let mut pending = states.map(Vec::into_iter);
323        for id in 0..config.workers {
324            let (command_tx, commands) = bounded(1);
325            let (credit_tx, credits) = bounded(config.prefetch_factor);
326            for _ in 0..config.prefetch_factor {
327                credit_tx.try_send(()).expect("fresh bounded credits");
328            }
329            pool.commands.push(command_tx);
330            pool.credits.push(credit_tx);
331            let source_factory = Arc::clone(&factory);
332            let transform_factory = Arc::clone(&transforms);
333            let run = pool.run.clone();
334            let shutdown = pool.shutdown.clone();
335            let committed = Arc::clone(&pool.committed);
336            let initialized = init_tx.clone();
337            let acknowledgments = ack_tx.clone();
338            let results = result_tx.clone();
339            let state = pending.as_mut().and_then(Iterator::next);
340            let info = WorkerInfo::from_loader_seed(
341                id,
342                config.workers,
343                config.loader_seed,
344                config.rank,
345                source_generation,
346            )?;
347            let mut journal = VecDeque::new();
348            journal
349                .try_reserve_exact(journal_capacity)
350                .map_err(|e| invalid(format!("snapshot journal unavailable: {e}")))?;
351            if journal.capacity() > journal_capacity {
352                return Err(
353                    invalid("snapshot journal exceeds validated retention capacity").into(),
354                );
355            }
356            let handle = thread::Builder::new().name(format!("rusttorch-exact-stream-{id}")).spawn(move || {
357                let mut did_initialize = false;
358                let mut generation = run.generation;
359                let outcome = catch_unwind(AssertUnwindSafe(|| with_worker_info(info, || {
360                    let context = WorkerContext::new(info, run.cancellation.clone(), run.deadline.clone());
361                    let mut source = source_factory.create(context.clone()).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })?;
362                    let mut transform = transform_factory.create(Some(&context)).map_err(|source| StreamCheckpointBuildError::TransformFactory { worker: id, source })?;
363                    if let Some(state) = &state {
364                        source.validate_snapshot(&state.source).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })?;
365                        transform.validate_snapshot(&state.transform).map_err(|source| StreamCheckpointBuildError::TransformState { worker: id, source })?;
366                    }
367                    if initialized.send((id, Ok(()))).is_err() { return Ok(()); }
368                    did_initialize = true;
369                    let mut pending = state;
370                    loop {
371                        let command = select! {
372                            recv(commands) -> command => match command { Ok(command) => command, Err(_) => return Ok(()) },
373                            recv(shutdown.signal()) -> _ => return Ok(()),
374                        };
375                        match command {
376                            Command::Apply => {
377                                if let Some(state) = pending.take() {
378                                    source.restore_validated(&state.source);
379                                    transform.restore_validated(&state.transform);
380                                }
381                            }
382                            Command::Begin(run) => {
383                                generation = run.generation;
384                                source.set_run_context(WorkerContext::new(info, run.cancellation.clone(), run.deadline.clone()));
385                                let mut previous = None;
386                                loop {
387                                    let acquired = select! {
388                                        recv(credits) -> result => result.is_ok(),
389                                        recv(run.cancellation.signal()) -> _ => false,
390                                        recv(shutdown.signal()) -> _ => false,
391                                    };
392                                    if !acquired || run.cancellation.is_cancelled() || shutdown.is_cancelled() { break; }
393                                    let boundary = committed.load(Ordering::Acquire);
394                                    while journal.front().is_some_and(|(attempt, _): &(Attempt, LaneState<S, F>)| attempt.sequence.is_some_and(|sequence| sequence < boundary)) { journal.pop_front(); }
395                                    assert!(journal.len() < journal_capacity, "validated stream snapshot window exhausted");
396                                    journal.push_back((Attempt { sequence: None, _logical_id: None, _outcome: AttemptOutcome::InFlight }, StreamLaneState { id, source: source.snapshot(), transform: transform.snapshot() }));
397                                    let event = match source.next() {
398                                        None => Event::End,
399                                        Some(Err(error)) => {
400                                            let sequence = source.error_sequence(&error).get();
401                                            if sequence < boundary || previous.is_some_and(|previous| sequence <= previous) {
402                                                Event::Protocol { sequence: Some(sequence), reason: "exact stream source error has a past or non-increasing sequence".to_owned() }
403                                            } else { Event::Source { sequence, error } }
404                                        },
405                                        Some(Ok(record)) => {
406                                            let sequence = record.sequence.map(SequenceId::get);
407                                            match sequence {
408                                                None => Event::Protocol { sequence, reason: "exact stream record is missing its sequence ID".to_owned() },
409                                                Some(sequence) if sequence < boundary || previous.is_some_and(|previous| sequence <= previous) => Event::Protocol { sequence: Some(sequence), reason: "exact stream shard emitted a past or non-increasing sequence".to_owned() },
410                                                Some(sequence) => {
411                                                    previous = Some(sequence);
412                                                    let logical_id = record.logical_id.get();
413                                                    let task = TaskContext { loader_seed: run.loader_seed, epoch: run.epoch, rank: info.rank, logical_sample: logical_id, stage: 0, cancellation: run.cancellation.clone(), deadline: run.deadline.clone() };
414                                                    match transform.transform(record.sample, &task) {
415                                                        Ok(sample) => Event::Record(WorkerRecord { sequence: record.sequence, logical_id: record.logical_id, sample }),
416                                                        Err(error) => Event::Transform { sequence, logical_id, error },
417                                                    }
418                                                }
419                                            }
420                                        }
421                                    };
422                                    let (sequence, logical_id, outcome) = match &event {
423                                        Event::Record(record) => (record.sequence.map(SequenceId::get), Some(record.logical_id.get()), AttemptOutcome::Record),
424                                        Event::End => (None, None, AttemptOutcome::End),
425                                        Event::Source { sequence, .. } => (Some(*sequence), None, AttemptOutcome::SourceError),
426                                        Event::Transform { sequence, logical_id, .. } => (Some(*sequence), Some(*logical_id), AttemptOutcome::TransformError),
427                                        Event::Protocol { sequence, .. } => (*sequence, None, AttemptOutcome::Protocol),
428                                        Event::Panic => unreachable!("panic is handled outside production"),
429                                    };
430                                    journal.back_mut().expect("pre-attempt snapshot").0 = Attempt { sequence, _logical_id: logical_id, _outcome: outcome };
431                                    let terminal = !matches!(event, Event::Record(_));
432                                    let message = Completion { worker: id, generation, event };
433                                    let sent = select! {
434                                        send(results, message) -> result => result.is_ok(),
435                                        recv(run.cancellation.signal()) -> _ => false,
436                                        recv(shutdown.signal()) -> _ => false,
437                                    };
438                                    if terminal || !sent { break; }
439                                }
440                            }
441                            Command::Rollback { generation: requested, boundary } => {
442                                let state = if requested != generation { Err(invalid("stale rollback generation").into()) } else {
443                                    let state = journal.iter().find(|(attempt, _)| attempt.sequence.is_none_or(|sequence| sequence >= boundary)).map(|(_, state)| state.clone())
444                                        .unwrap_or_else(|| StreamLaneState { id, source: source.snapshot(), transform: transform.snapshot() });
445                                    source.validate_snapshot(&state.source).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })
446                                        .and_then(|()| transform.validate_snapshot(&state.transform).map_err(|source| StreamCheckpointBuildError::TransformState { worker: id, source }))
447                                        .map(|()| { source.restore_validated(&state.source); transform.restore_validated(&state.transform); state })
448                                };
449                                journal.clear();
450                                while credits.try_recv().is_ok() {}
451                                if acknowledgments.send((id, generation, state)).is_err() { return Ok(()); }
452                            }
453                        }
454                    }
455                })));
456                match outcome {
457                    Ok(Err(error)) => { let _ = initialized.send((id, Err(error))); }
458                    Err(_) if !did_initialize => { let _ = initialized.send((id, Err(StreamCheckpointBuildError::WorkerPanic { worker: id }))); }
459                    Err(_) => {
460                        let message = Completion { worker: id, generation, event: Event::Panic };
461                        select! { send(results, message) -> _ => {}, recv(shutdown.signal()) -> _ => {} }
462                    }
463                    Ok(Ok(())) => {}
464                }
465            }).map_err(|e| invalid(format!("cannot start exact stream worker: {e}")))?;
466            pool.handles.push(handle);
467        }
468        drop(init_tx);
469        drop(ack_tx);
470        drop(result_tx);
471        let mut seen = vec![false; config.workers];
472        for _ in 0..config.workers {
473            let (id, result) = pool
474                .initialized
475                .recv()
476                .map_err(|_| invalid("worker closed during validation"))?;
477            if id >= seen.len() || std::mem::replace(&mut seen[id], true) {
478                return Err(invalid("duplicate initialization lane").into());
479            }
480            result?;
481        }
482        Ok(pool)
483    }
484
485    fn begin(&self) -> Result<()> {
486        for sender in &self.commands {
487            sender
488                .send(Command::Begin(self.run.clone()))
489                .map_err(|_| invalid("worker closed before production"))?;
490        }
491        Ok(())
492    }
493
494    fn apply(&self) -> Result<()> {
495        for sender in &self.commands {
496            sender
497                .send(Command::Apply)
498                .map_err(|_| invalid("worker closed before validated apply"))?;
499        }
500        Ok(())
501    }
502
503    fn rollback(
504        &mut self,
505        boundary: u64,
506        prefetch: usize,
507    ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
508        let generation = self.run.generation;
509        let next_generation = generation
510            .checked_add(1)
511            .filter(|g| *g < u64::MAX)
512            .ok_or_else(|| invalid("transport generation overflow"))?;
513        self.run.cancel();
514        for command in &self.commands {
515            command
516                .send(Command::Rollback {
517                    generation,
518                    boundary,
519                })
520                .map_err(|_| invalid("worker failed before rollback"))?;
521        }
522        let states = (0..self.commands.len()).map(|_| None).collect::<Vec<_>>();
523        self.collect_rollback(states, generation, next_generation, boundary, prefetch)
524    }
525
526    fn collect_rollback(
527        &mut self,
528        mut states: Vec<Option<LaneState<S, F>>>,
529        generation: u64,
530        next_generation: u64,
531        boundary: u64,
532        prefetch: usize,
533    ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
534        while states.iter().any(Option::is_none) {
535            select! {
536                recv(self.acknowledgments) -> reply => {
537                    let (id, received, state) = reply.map_err(|_| invalid("missing rollback acknowledgment"))?;
538                    if id >= states.len() || received != generation || states[id].is_some() { return Err(invalid("stale or duplicate rollback acknowledgment").into()); }
539                    states[id] = Some(state?);
540                },
541                recv(self.results) -> message => {
542                    if matches!(message, Ok(Completion { event: Event::Panic, .. }) | Err(_)) { return Err(invalid("worker failed before rollback acknowledgment").into()); }
543                },
544            }
545        }
546        self.finish_rollback(states, next_generation, boundary, prefetch)
547    }
548
549    fn finish_rollback(
550        &mut self,
551        states: Vec<Option<LaneState<S, F>>>,
552        generation: u64,
553        boundary: u64,
554        prefetch: usize,
555    ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
556        if self.acknowledgments.try_recv().is_ok() {
557            return Err(invalid("duplicate rollback acknowledgment").into());
558        }
559        while self.results.try_recv().is_ok() {}
560        for credit in &self.credits {
561            for _ in 0..prefetch {
562                credit
563                    .try_send(())
564                    .map_err(|_| invalid("rollback credits were not drained"))?;
565            }
566        }
567        self.committed.store(boundary, Ordering::Release);
568        self.run = WorkerRunContext::new(generation, self.run.loader_seed, self.run.epoch);
569        Ok(states.into_iter().map(Option::unwrap).collect())
570    }
571}
572
573/// Exact stream owner with suspended, transactionally validated worker sources.
574pub struct ExactStreamDataLoader<S, C, F = IdentityTransformFactory, N = PinDisabled>
575where
576    S: WorkerSourceFactory,
577    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
578    F: TransformFactory<S::Sample>,
579    F::Transform: WorkerCheckpoint,
580    C: Collate<TransformOutput<S, F>> + Checkpointable,
581{
582    factory: Arc<S>,
583    transform_factory: Arc<F>,
584    collator: C,
585    configuration: StreamConfiguration,
586    identity: String,
587    pin: PinMemoryStatus,
588    length: Option<usize>,
589    pending: Option<Pool<S, F>>,
590    next_batch: u64,
591    next_sequence: u64,
592    short_tail: bool,
593    source_generation: u64,
594    next_generation: u64,
595    policies: PhantomData<N>,
596}
597
598impl<S, C, F, N> StreamDataLoaderBuilder<S, C, F, NoWorkerInit, MemoryDisabled, N, CheckpointFresh>
599where
600    S: CheckpointSourceFactory,
601    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
602    SourceState<S>: Send + 'static,
603    F: TransformFactory<S::Sample> + Send + Sync + 'static,
604    F::Transform: WorkerCheckpoint + Send + 'static,
605    TransformState<S, F>: Send + 'static,
606    TransformOutput<S, F>: Send + 'static,
607    TransformFailure<S, F>: Send + 'static,
608    F::Error: Send + 'static,
609    C: Collate<TransformOutput<S, F>> + Checkpointable,
610    N: StreamPinPolicy<S, C, F>,
611{
612    /// Validates exact capabilities and constructs suspended worker instances.
613    pub fn build(self) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>> {
614        let identity = self.checkpoint.identity.clone();
615        build(self, identity, None)
616    }
617}
618
619impl<S, C, F, N>
620    StreamDataLoaderBuilder<
621        S,
622        C,
623        F,
624        NoWorkerInit,
625        MemoryDisabled,
626        N,
627        CheckpointResume<State<S, F, C>>,
628    >
629where
630    S: CheckpointSourceFactory,
631    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
632    SourceState<S>: Send + 'static,
633    F: TransformFactory<S::Sample> + Send + Sync + 'static,
634    F::Transform: WorkerCheckpoint + Send + 'static,
635    TransformState<S, F>: Send + 'static,
636    TransformOutput<S, F>: Send + 'static,
637    TransformFailure<S, F>: Send + 'static,
638    F::Error: Send + 'static,
639    C: Collate<TransformOutput<S, F>> + Checkpointable,
640    N: StreamPinPolicy<S, C, F>,
641{
642    /// Validates the envelope before callbacks, then all components before apply.
643    pub fn build(self) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>> {
644        let StreamDataLoaderBuilder {
645            factory,
646            collator,
647            transform_factory,
648            worker_init,
649            configuration,
650            checkpoint,
651            ..
652        } = self;
653        let builder = StreamDataLoaderBuilder {
654            factory,
655            collator,
656            transform_factory,
657            worker_init,
658            configuration,
659            checkpoint: CheckpointDisabled,
660            states: PhantomData,
661        };
662        build(builder, checkpoint.identity, Some(checkpoint.state))
663    }
664}
665
666fn build<S, C, F, N, K>(
667    builder: StreamDataLoaderBuilder<S, C, F, NoWorkerInit, MemoryDisabled, N, K>,
668    identity: String,
669    state: Option<State<S, F, C>>,
670) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>>
671where
672    S: CheckpointSourceFactory,
673    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
674    SourceState<S>: Send + 'static,
675    F: TransformFactory<S::Sample> + Send + Sync + 'static,
676    F::Transform: WorkerCheckpoint + Send + 'static,
677    TransformState<S, F>: Send + 'static,
678    TransformOutput<S, F>: Send + 'static,
679    TransformFailure<S, F>: Send + 'static,
680    F::Error: Send + 'static,
681    C: Collate<TransformOutput<S, F>> + Checkpointable,
682    N: StreamPinPolicy<S, C, F>,
683{
684    let config = builder.configuration;
685    validate_capacity::<S, F>(config)?;
686    if identity.is_empty() || S::CHECKPOINT_KIND.is_empty() {
687        return Err(invalid("source identity and factory kind must not be empty").into());
688    }
689    let pin = resolve_stream_pin(config.pin_memory)?;
690    if let Some(state) = &state {
691        let batch_size = u64::try_from(config.batch_size)
692            .map_err(|_| invalid("batch size exceeds sequence range"))?;
693        let full = state
694            .next_batch
695            .checked_mul(batch_size)
696            .ok_or_else(|| invalid("batch cursor overflow"))?;
697        let short_tail = !config.drop_last
698            && state.next_batch > 0
699            && state.next_sequence < full
700            && state.next_sequence > full - batch_size;
701        if state.schema_version != 1
702            || state.source_identity != identity
703            || state.factory_kind != S::CHECKPOINT_KIND
704            || state.epoch != config.epoch
705            || state.source_generation > state.transport_generation
706            || state.transport_generation >= u64::MAX - 1
707            || state.rng_derivation_version != TASK_RNG_DERIVATION_VERSION
708            || state.worker_seed_derivation_version != WORKER_SEED_DERIVATION_VERSION
709            || state.configuration != configuration(config, pin)
710            || state.lanes.len() != config.workers
711            || state
712                .lanes
713                .iter()
714                .enumerate()
715                .any(|(id, lane)| lane.id != id)
716            || state.short_tail != short_tail
717            || (state.next_sequence != full && !short_tail)
718            || state.source_length.is_some_and(|length| {
719                state.next_sequence > length as u64
720                    || (short_tail && state.next_sequence != length as u64)
721            })
722        {
723            return Err(invalid("incompatible or corrupt stream checkpoint envelope").into());
724        }
725    }
726    let length = builder.factory.exact_len();
727    if state
728        .as_ref()
729        .is_some_and(|state| state.source_length != length)
730    {
731        return Err(invalid("source length changed").into());
732    }
733    let mut loader = ExactStreamDataLoader {
734        factory: Arc::new(builder.factory),
735        transform_factory: Arc::new(builder.transform_factory),
736        collator: builder.collator,
737        configuration: config,
738        identity,
739        pin,
740        length,
741        pending: None,
742        next_batch: state.as_ref().map_or(0, |s| s.next_batch),
743        next_sequence: state.as_ref().map_or(0, |s| s.next_sequence),
744        short_tail: state.as_ref().is_some_and(|s| s.short_tail),
745        source_generation: state.as_ref().map_or(0, |s| s.source_generation),
746        next_generation: state.as_ref().map_or(1, |s| s.transport_generation + 1),
747        policies: PhantomData,
748    };
749    let generation = state.as_ref().map_or(0, |s| s.transport_generation);
750    let pool = Pool::new(
751        Arc::clone(&loader.factory),
752        Arc::clone(&loader.transform_factory),
753        config,
754        loader.source_generation,
755        WorkerRunContext::new(generation, config.loader_seed, config.epoch),
756        loader.next_sequence,
757        state.as_ref().map(|s| s.lanes.clone()),
758    )?;
759    if let Some(state) = &state {
760        loader.collator.validate_state(&state.coordinator)?;
761        pool.apply()?;
762        loader.collator.load_validated(&state.coordinator);
763    }
764    loader.pending = Some(pool);
765    Ok(loader)
766}
767
768/// Iterator whose checkpoint rolls unpublished reads back to the visible boundary.
769pub struct ExactStreamLoaderIter<'a, S, C, F = IdentityTransformFactory, N = PinDisabled>
770where
771    S: WorkerSourceFactory,
772    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
773    F: TransformFactory<S::Sample>,
774    F::Transform: WorkerCheckpoint,
775    C: Collate<TransformOutput<S, F>> + Checkpointable,
776{
777    loader: &'a mut ExactStreamDataLoader<S, C, F, N>,
778    pool: Option<Pool<S, F>>,
779    completed: Vec<Buffered<S, F>>,
780    ended: Vec<bool>,
781    next_sequence: u64,
782    visible_sequence: u64,
783    short_tail: bool,
784    next_batch: u64,
785    stopped: bool,
786    boundary_valid: bool,
787    pending_error: Option<IterError<S, F, C>>,
788}
789
790impl<S, C, F, N> ExactStreamDataLoader<S, C, F, N>
791where
792    S: CheckpointSourceFactory,
793    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
794    SourceState<S>: Send + 'static,
795    F: TransformFactory<S::Sample> + Send + Sync + 'static,
796    F::Transform: WorkerCheckpoint + Send + 'static,
797    TransformState<S, F>: Send + 'static,
798    TransformOutput<S, F>: Send + 'static,
799    TransformFailure<S, F>: Send + 'static,
800    F::Error: Send + 'static,
801    C: Collate<TransformOutput<S, F>> + Checkpointable,
802    N: StreamPinPolicy<S, C, F>,
803{
804    /// Returns the exact global batch count, when known.
805    pub fn len(&self) -> Option<usize> {
806        self.length.map(|n| {
807            if self.configuration.drop_last {
808                n / self.configuration.batch_size
809            } else {
810                n.div_ceil(self.configuration.batch_size)
811            }
812        })
813    }
814    /// Returns whether the known batch count is zero.
815    pub fn is_empty(&self) -> Option<bool> {
816        self.len().map(|n| n == 0)
817    }
818    /// Returns the shard count.
819    pub fn workers(&self) -> usize {
820        self.configuration.workers
821    }
822    /// Returns the effective pinning policy.
823    pub fn pin_memory_status(&self) -> PinMemoryStatus {
824        self.pin
825    }
826    /// Selects a fresh epoch and discards a retained resume.
827    pub fn set_epoch(&mut self, epoch: u64) {
828        self.pending = None;
829        self.configuration.epoch = epoch;
830    }
831    /// Starts the retained resume once, then fresh logical generations.
832    pub fn iter(&mut self) -> ExactStreamLoaderIter<'_, S, C, F, N> {
833        let retained = self.pending.is_some();
834        let mut pending_error = None;
835        let pool = if retained {
836            self.pending.take()
837        } else {
838            let generation = self.next_generation;
839            if let Some(next) = generation.checked_add(1).filter(|g| *g < u64::MAX) {
840                self.next_generation = next;
841                self.source_generation = generation;
842                match Pool::new(
843                    Arc::clone(&self.factory),
844                    Arc::clone(&self.transform_factory),
845                    self.configuration,
846                    generation,
847                    WorkerRunContext::new(
848                        generation,
849                        self.configuration.loader_seed,
850                        self.configuration.epoch,
851                    ),
852                    0,
853                    None,
854                ) {
855                    Ok(pool) => Some(pool),
856                    Err(error) => {
857                        pending_error = Some(map_build_error::<S, F, C>(error));
858                        None
859                    }
860                }
861            } else {
862                pending_error = Some(LoaderError::Configuration(invalid("generation overflow")));
863                None
864            }
865        };
866        if let Some(pool) = &pool
867            && let Err(error) = pool.begin()
868        {
869            pending_error = Some(LoaderError::Configuration(error));
870        }
871        let next_batch = if retained { self.next_batch } else { 0 };
872        let next_sequence = if retained { self.next_sequence } else { 0 };
873        let short_tail = retained && self.short_tail;
874        let ended = vec![false; self.configuration.workers];
875        let completed =
876            Vec::with_capacity(self.configuration.workers * self.configuration.prefetch_factor);
877        ExactStreamLoaderIter {
878            loader: self,
879            pool,
880            completed,
881            ended,
882            next_batch,
883            next_sequence,
884            visible_sequence: next_sequence,
885            short_tail,
886            stopped: false,
887            boundary_valid: pending_error.is_none(),
888            pending_error,
889        }
890    }
891}
892
893fn map_build_error<S, F, C>(error: BuildError<S, F>) -> IterError<S, F, C>
894where
895    S: WorkerSourceFactory,
896    F: TransformFactory<S::Sample>,
897    C: Collate<TransformOutput<S, F>>,
898{
899    match error {
900        StreamCheckpointBuildError::Configuration(e) => LoaderError::Configuration(e),
901        StreamCheckpointBuildError::Source { worker, source } => LoaderError::StreamPipeline {
902            batch: None,
903            worker,
904            sequence: None,
905            logical_id: None,
906            source: PipelineError::Source(source),
907        },
908        StreamCheckpointBuildError::TransformFactory { worker, source } => {
909            LoaderError::StreamPipeline {
910                batch: None,
911                worker,
912                sequence: None,
913                logical_id: None,
914                source: PipelineError::TransformInit(source),
915            }
916        }
917        StreamCheckpointBuildError::TransformState { source, .. } => {
918            LoaderError::Configuration(source)
919        }
920        StreamCheckpointBuildError::WorkerPanic { worker } => LoaderError::WorkerPanic {
921            worker,
922            batch: None,
923        },
924    }
925}
926
927impl<S, C, F, N> ExactStreamLoaderIter<'_, S, C, F, N>
928where
929    S: CheckpointSourceFactory,
930    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
931    SourceState<S>: Send + 'static,
932    F: TransformFactory<S::Sample> + Send + Sync + 'static,
933    F::Transform: WorkerCheckpoint + Send + 'static,
934    TransformState<S, F>: Send + 'static,
935    TransformOutput<S, F>: Send + 'static,
936    TransformFailure<S, F>: Send + 'static,
937    F::Error: Send + 'static,
938    C: Collate<TransformOutput<S, F>> + Checkpointable,
939    N: StreamPinPolicy<S, C, F>,
940{
941    /// Cancels active callbacks and rolls paired snapshots back before resuming.
942    ///
943    /// The original iterator remains usable. Source/transform errors can be
944    /// replayed from the last successful boundary; coordinator failures cannot.
945    /// Blocking native callbacks must cooperate with cancellation to finish.
946    pub fn checkpoint(&mut self) -> std::result::Result<State<S, F, C>, IterError<S, F, C>> {
947        if !self.boundary_valid {
948            return Err(LoaderError::Checkpoint {
949                reason: "no recoverable visible stream boundary".to_owned(),
950            });
951        }
952        let coordinator = self.loader.collator.save_state();
953        let pool = self.pool.as_mut().ok_or_else(|| LoaderError::Checkpoint {
954            reason: "missing worker pool".to_owned(),
955        })?;
956        let lanes = match pool.rollback(
957            self.visible_sequence,
958            self.loader.configuration.prefetch_factor,
959        ) {
960            Ok(lanes) => lanes,
961            Err(error) => {
962                self.boundary_valid = false;
963                self.stopped = true;
964                self.pool = None;
965                return Err(map_build_error::<S, F, C>(error));
966            }
967        };
968        self.completed.clear();
969        self.ended.fill(false);
970        self.next_sequence = self.visible_sequence;
971        self.stopped = false;
972        let state = StreamLoaderState {
973            schema_version: 1,
974            source_identity: self.loader.identity.clone(),
975            factory_kind: S::CHECKPOINT_KIND.to_owned(),
976            epoch: self.loader.configuration.epoch,
977            source_generation: self.loader.source_generation,
978            transport_generation: pool.run.generation,
979            next_batch: self.next_batch,
980            next_sequence: self.visible_sequence,
981            short_tail: self.short_tail,
982            source_length: self.loader.length,
983            rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
984            worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
985            configuration: configuration(self.loader.configuration, self.loader.pin),
986            lanes,
987            coordinator,
988        };
989        self.loader.next_generation = self.loader.next_generation.max(pool.run.generation + 1);
990        pool.begin().map_err(LoaderError::Configuration)?;
991        Ok(state)
992    }
993
994    fn fail(
995        &mut self,
996        error: IterError<S, F, C>,
997        recoverable: bool,
998    ) -> Option<std::result::Result<C::Batch, IterError<S, F, C>>> {
999        self.stopped = true;
1000        self.boundary_valid &= recoverable;
1001        if let Some(pool) = &self.pool {
1002            pool.run.cancel();
1003        }
1004        if !recoverable {
1005            self.pool = None;
1006        }
1007        Some(Err(error))
1008    }
1009}
1010
1011impl<S, C, F, N> Iterator for ExactStreamLoaderIter<'_, S, C, F, N>
1012where
1013    S: CheckpointSourceFactory,
1014    S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
1015    SourceState<S>: Send + 'static,
1016    F: TransformFactory<S::Sample> + Send + Sync + 'static,
1017    F::Transform: WorkerCheckpoint + Send + 'static,
1018    TransformState<S, F>: Send + 'static,
1019    TransformOutput<S, F>: Send + 'static,
1020    TransformFailure<S, F>: Send + 'static,
1021    F::Error: Send + 'static,
1022    C: Collate<TransformOutput<S, F>> + Checkpointable,
1023    N: StreamPinPolicy<S, C, F>,
1024{
1025    type Item = std::result::Result<C::Batch, IterError<S, F, C>>;
1026    fn next(&mut self) -> Option<Self::Item> {
1027        if self.stopped {
1028            return None;
1029        }
1030        if let Some(error) = self.pending_error.take() {
1031            return self.fail(error, false);
1032        }
1033        let mut samples = Vec::with_capacity(self.loader.configuration.batch_size);
1034        loop {
1035            if let Some(index) = self
1036                .completed
1037                .iter()
1038                .position(|(_, event)| event.sequence() == Some(self.next_sequence))
1039            {
1040                let (worker, event) = self.completed.swap_remove(index);
1041                let record = match event {
1042                    Event::Record(record) => record,
1043                    Event::Source { sequence, error } => {
1044                        return self.fail(
1045                            LoaderError::StreamPipeline {
1046                                batch: Some(self.next_batch),
1047                                worker,
1048                                sequence: Some(sequence),
1049                                logical_id: None,
1050                                source: PipelineError::Source(error),
1051                            },
1052                            true,
1053                        );
1054                    }
1055                    Event::Transform {
1056                        sequence,
1057                        logical_id,
1058                        error,
1059                    } => {
1060                        return self.fail(
1061                            LoaderError::StreamPipeline {
1062                                batch: Some(self.next_batch),
1063                                worker,
1064                                sequence: Some(sequence),
1065                                logical_id: Some(logical_id),
1066                                source: PipelineError::Transform(error),
1067                            },
1068                            true,
1069                        );
1070                    }
1071                    _ => unreachable!("only sequenced outcomes enter reassembly"),
1072                };
1073                let Some(next) = self.next_sequence.checked_add(1) else {
1074                    return self.fail(
1075                        LoaderError::StreamProtocol {
1076                            sequence: Some(self.next_sequence),
1077                            reason: "sequence overflow".to_owned(),
1078                        },
1079                        false,
1080                    );
1081                };
1082                self.next_sequence = next;
1083                samples.push(record.sample);
1084                if self.pool.as_ref().expect("active pool").credits[worker]
1085                    .try_send(())
1086                    .is_err()
1087                {
1088                    return self.fail(
1089                        LoaderError::ChannelClosed {
1090                            batch: self.next_batch,
1091                        },
1092                        false,
1093                    );
1094                }
1095                if samples.len() == self.loader.configuration.batch_size {
1096                    break;
1097                }
1098                continue;
1099            }
1100            if self.ended.iter().all(|ended| *ended) {
1101                if !self.completed.is_empty()
1102                    || self
1103                        .loader
1104                        .length
1105                        .is_some_and(|length| self.next_sequence != length as u64)
1106                {
1107                    return self.fail(
1108                        LoaderError::StreamProtocol {
1109                            sequence: Some(self.next_sequence),
1110                            reason:
1111                                "stream ended with a missing global sequence or wrong exact length"
1112                                    .to_owned(),
1113                        },
1114                        false,
1115                    );
1116                }
1117                if samples.is_empty() || self.loader.configuration.drop_last {
1118                    self.stopped = true;
1119                    return None;
1120                }
1121                break;
1122            }
1123            // ponytail: bounded lane/window scan; retain lane counts if wide sharding matters.
1124            if self.ended.iter().enumerate().all(|(id, ended)| {
1125                *ended
1126                    || self
1127                        .completed
1128                        .iter()
1129                        .filter(|(worker, _)| *worker == id)
1130                        .count()
1131                        == self.loader.configuration.prefetch_factor
1132            }) {
1133                return self.fail(
1134                    LoaderError::StreamProtocol {
1135                        sequence: Some(self.next_sequence),
1136                        reason: "bounded stream window contains no expected sequence".to_owned(),
1137                    },
1138                    false,
1139                );
1140            }
1141            let pool = self.pool.as_ref().expect("active pool");
1142            let completion = match pool.results.recv() {
1143                Ok(completion) => completion,
1144                Err(_) => {
1145                    return self.fail(
1146                        LoaderError::ChannelClosed {
1147                            batch: self.next_batch,
1148                        },
1149                        false,
1150                    );
1151                }
1152            };
1153            if completion.generation != pool.run.generation {
1154                return self.fail(
1155                    LoaderError::StreamProtocol {
1156                        sequence: None,
1157                        reason: "stale transport message".to_owned(),
1158                    },
1159                    false,
1160                );
1161            }
1162            let worker = completion.worker;
1163            match completion.event {
1164                event @ (Event::Record(_) | Event::Source { .. } | Event::Transform { .. }) => {
1165                    let sequence = event.sequence();
1166                    if sequence.is_none_or(|sequence| sequence < self.next_sequence)
1167                        || self
1168                            .completed
1169                            .iter()
1170                            .any(|(_, previous)| previous.sequence() == sequence)
1171                    {
1172                        return self.fail(
1173                            LoaderError::StreamProtocol {
1174                                sequence,
1175                                reason: "missing, duplicate, or past sequence ID".to_owned(),
1176                            },
1177                            false,
1178                        );
1179                    }
1180                    if !matches!(event, Event::Record(_)) {
1181                        self.ended[worker] = true;
1182                    }
1183                    self.completed.push((worker, event));
1184                }
1185                Event::End => {
1186                    self.ended[worker] = true;
1187                }
1188                Event::Protocol { sequence, reason } => {
1189                    return self.fail(LoaderError::StreamProtocol { sequence, reason }, false);
1190                }
1191                Event::Panic => {
1192                    return self.fail(
1193                        LoaderError::StreamWorkerPanic {
1194                            batch: Some(self.next_batch),
1195                            worker,
1196                            sequence: None,
1197                            logical_id: None,
1198                        },
1199                        false,
1200                    );
1201                }
1202            }
1203        }
1204        let short_tail = samples.len() < self.loader.configuration.batch_size;
1205        let batch = match catch_unwind(AssertUnwindSafe(|| self.loader.collator.collate(samples))) {
1206            Ok(Ok(batch)) => batch,
1207            Ok(Err(source)) => {
1208                return self.fail(
1209                    LoaderError::Pipeline {
1210                        batch: Some(self.next_batch),
1211                        worker: None,
1212                        source: PipelineError::Collate(source),
1213                    },
1214                    false,
1215                );
1216            }
1217            Err(_) => {
1218                return self.fail(
1219                    LoaderError::CoordinatorPanic {
1220                        stage: "collation",
1221                        batch: Some(self.next_batch),
1222                    },
1223                    false,
1224                );
1225            }
1226        };
1227        let batch = if let PinMemoryStatus::Enabled(device) = self.loader.pin {
1228            match catch_unwind(AssertUnwindSafe(|| N::pin(batch, device))) {
1229                Ok(Ok(batch)) => batch,
1230                Ok(Err(source)) => {
1231                    return self.fail(
1232                        LoaderError::PinMemory {
1233                            batch: self.next_batch,
1234                            source,
1235                        },
1236                        false,
1237                    );
1238                }
1239                Err(_) => {
1240                    return self.fail(
1241                        LoaderError::CoordinatorPanic {
1242                            stage: "pinning",
1243                            batch: Some(self.next_batch),
1244                        },
1245                        false,
1246                    );
1247                }
1248            }
1249        } else {
1250            batch
1251        };
1252        let Some(next_batch) = self.next_batch.checked_add(1) else {
1253            return self.fail(
1254                LoaderError::Configuration(invalid("batch cursor overflow")),
1255                false,
1256            );
1257        };
1258        self.next_batch = next_batch;
1259        self.visible_sequence = self.next_sequence;
1260        self.short_tail = short_tail;
1261        self.pool
1262            .as_ref()
1263            .expect("active pool")
1264            .committed
1265            .store(self.visible_sequence, Ordering::Release);
1266        Some(Ok(batch))
1267    }
1268}