1use super::*;
4use crate::{
5 CheckpointDisabled, CheckpointFresh, CheckpointResume, CheckpointSourceFactory, Checkpointable,
6 CheckpointableSource, StreamCheckpointBuildError, StreamCheckpointConfiguration,
7 StreamLaneState, StreamLoaderState, TASK_RNG_DERIVATION_VERSION,
8 WORKER_SEED_DERIVATION_VERSION, WorkerCheckpoint,
9};
10use std::collections::VecDeque;
11use std::sync::atomic::{AtomicU64, Ordering};
12
13type SourceState<S> = <<S as WorkerSourceFactory>::Source as CheckpointableSource>::State;
14type TransformState<S, F> = <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as WorkerCheckpoint>::State;
15type LaneState<S, F> = StreamLaneState<SourceState<S>, TransformState<S, F>>;
16type State<S, F, C> =
17 StreamLoaderState<SourceState<S>, TransformState<S, F>, <C as Checkpointable>::State>;
18type BuildError<S, F> = StreamCheckpointBuildError<
19 <S as WorkerSourceFactory>::Error,
20 <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
21>;
22type IterError<S, F, C> = StreamLoaderError<S, C, F, NoWorkerInit>;
23
24fn invalid(reason: impl Into<String>) -> RustTorchError {
25 invalid_configuration("stream_checkpoint", reason)
26}
27
28impl<S, C, F, I, M, N> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointDisabled> {
29 pub fn checkpointable(
47 self,
48 identity: impl Into<String>,
49 ) -> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointFresh> {
50 StreamDataLoaderBuilder {
51 factory: self.factory,
52 collator: self.collator,
53 transform_factory: self.transform_factory,
54 worker_init: self.worker_init,
55 configuration: self.configuration,
56 states: PhantomData,
57 checkpoint: CheckpointFresh {
58 identity: identity.into(),
59 },
60 }
61 }
62
63 #[allow(clippy::type_complexity)]
71 pub fn resume<SS, TS, CS>(
72 self,
73 identity: impl Into<String>,
74 state: StreamLoaderState<SS, TS, CS>,
75 ) -> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointResume<StreamLoaderState<SS, TS, CS>>>
76 {
77 StreamDataLoaderBuilder {
78 factory: self.factory,
79 collator: self.collator,
80 transform_factory: self.transform_factory,
81 worker_init: self.worker_init,
82 configuration: self.configuration,
83 states: PhantomData,
84 checkpoint: CheckpointResume {
85 identity: identity.into(),
86 state,
87 },
88 }
89 }
90}
91
92fn configuration(
93 config: StreamConfiguration,
94 pin: PinMemoryStatus,
95) -> StreamCheckpointConfiguration {
96 StreamCheckpointConfiguration {
97 workers: config.workers,
98 prefetch_factor: config.prefetch_factor,
99 batch_size: config.batch_size,
100 drop_last: config.drop_last,
101 loader_seed: config.loader_seed,
102 rank: config.rank,
103 pin_request: match config.pin_memory {
104 StreamPinRequest::Disabled => crate::CheckpointPinRequest::Disabled,
105 StreamPinRequest::Auto => crate::CheckpointPinRequest::Auto,
106 StreamPinRequest::Explicit(Device::Cuda(index)) => {
107 crate::CheckpointPinRequest::ExplicitCuda { index }
108 }
109 _ => unreachable!("pin request validated"),
110 },
111 pin_status: match pin {
112 PinMemoryStatus::Disabled => crate::CheckpointPinStatus::Disabled,
113 PinMemoryStatus::DisabledNoAccelerator => {
114 crate::CheckpointPinStatus::DisabledNoAccelerator
115 }
116 PinMemoryStatus::Enabled(Device::Cuda(index)) => {
117 crate::CheckpointPinStatus::Cuda { index }
118 }
119 _ => unreachable!("pin status validated"),
120 },
121 }
122}
123
124enum Command {
125 Apply,
126 Begin(WorkerRunContext),
127 Rollback { generation: u64, boundary: u64 },
128}
129
130enum Event<T, SE, TE> {
131 Record(WorkerRecord<T>),
132 End,
133 Source {
134 sequence: u64,
135 error: SE,
136 },
137 Transform {
138 sequence: u64,
139 logical_id: u64,
140 error: TE,
141 },
142 Protocol {
143 sequence: Option<u64>,
144 reason: String,
145 },
146 Panic,
147}
148
149impl<T, SE, TE> Event<T, SE, TE> {
150 fn sequence(&self) -> Option<u64> {
151 match self {
152 Self::Record(record) => record.sequence.map(SequenceId::get),
153 Self::Source { sequence, .. } | Self::Transform { sequence, .. } => Some(*sequence),
154 _ => None,
155 }
156 }
157}
158
159type Buffered<S, F> = (
160 usize,
161 Event<TransformOutput<S, F>, <S as WorkerSourceFactory>::Error, TransformFailure<S, F>>,
162);
163
164enum AttemptOutcome {
165 InFlight,
166 Record,
167 End,
168 SourceError,
169 TransformError,
170 Protocol,
171}
172
173struct Attempt {
174 sequence: Option<u64>,
175 _logical_id: Option<u64>,
176 _outcome: AttemptOutcome,
177}
178
179struct Completion<T, SE, TE> {
180 worker: usize,
181 generation: u64,
182 event: Event<T, SE, TE>,
183}
184type CompletionFor<S, F> =
185 Completion<TransformOutput<S, F>, <S as WorkerSourceFactory>::Error, TransformFailure<S, F>>;
186type Ack<S, F> = (
187 usize,
188 u64,
189 std::result::Result<LaneState<S, F>, BuildError<S, F>>,
190);
191
192struct Pool<S, F>
193where
194 S: WorkerSourceFactory,
195 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
196 F: TransformFactory<S::Sample>,
197 F::Transform: WorkerCheckpoint,
198{
199 commands: Vec<Sender<Command>>,
200 credits: Vec<Sender<()>>,
201 results: Receiver<CompletionFor<S, F>>,
202 #[allow(clippy::type_complexity)]
203 initialized: Receiver<(usize, std::result::Result<(), BuildError<S, F>>)>,
204 acknowledgments: Receiver<Ack<S, F>>,
205 handles: Vec<JoinHandle<()>>,
206 shutdown: CancellationToken,
207 run: WorkerRunContext,
208 committed: Arc<AtomicU64>,
209}
210
211impl<S, F> Drop for Pool<S, F>
212where
213 S: WorkerSourceFactory,
214 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
215 F: TransformFactory<S::Sample>,
216 F::Transform: WorkerCheckpoint,
217{
218 fn drop(&mut self) {
219 self.run.cancel();
220 self.shutdown.cancel();
221 for handle in self.handles.drain(..) {
222 let _ = handle.join();
223 }
224 }
225}
226
227fn validate_capacity<S, F>(config: StreamConfiguration) -> Result<usize>
228where
229 S: WorkerSourceFactory,
230 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
231 F: TransformFactory<S::Sample>,
232 F::Transform: WorkerCheckpoint,
233{
234 if config.workers == 0 || config.prefetch_factor == 0 || config.batch_size == 0 {
235 return Err(invalid(
236 "workers, prefetch_factor, and batch_size must be positive",
237 ));
238 }
239 if !config.ordered
240 || config.persistent_workers
241 || config.timeout.is_some()
242 || config.prefetch_bytes.is_some()
243 {
244 return Err(invalid(
245 "exact streams require ordered nonpersistent workers without timeout or byte accounting",
246 ));
247 }
248 let outstanding = config
249 .workers
250 .checked_mul(config.prefetch_factor)
251 .ok_or_else(|| invalid("outstanding overflow"))?;
252 let journal = config
255 .prefetch_factor
256 .checked_add(config.batch_size)
257 .and_then(|n| n.checked_add(1))
258 .ok_or_else(|| invalid("journal window overflow"))?;
259 let extra = config
260 .workers
261 .checked_mul(journal)
262 .and_then(|n| n.checked_mul(size_of::<(Attempt, LaneState<S, F>)>()))
263 .and_then(|n| {
264 n.checked_add(
265 config.workers.checked_mul(
266 size_of::<LaneState<S, F>>()
267 .checked_mul(8)?
268 .checked_add(8192)?
269 .checked_add(size_of::<CapacitySlot<Ack<S, F>>>())?
270 .checked_add(size_of::<
271 CapacitySlot<(usize, std::result::Result<(), BuildError<S, F>>)>,
272 >())?,
273 )?,
274 )
275 })
276 .and_then(|n| {
277 n.checked_add(outstanding.checked_mul(size_of::<CapacitySlot<CompletionFor<S, F>>>())?)
278 })
279 .and_then(|n| n.checked_add(outstanding.checked_mul(size_of::<Buffered<S, F>>())?))
280 .ok_or_else(|| invalid("snapshot storage overflow"))?;
281 validate_stream_capacity_extra::<S, F, NoWorkerInit, MemoryDisabled>(
282 config.workers,
283 config.prefetch_factor,
284 outstanding,
285 config.batch_size,
286 outstanding,
287 extra,
288 )?;
289 preflight_vec::<(Attempt, LaneState<S, F>)>(journal, "stream checkpoint journal")?;
290 Ok(journal)
291}
292
293impl<S, F> Pool<S, F>
294where
295 S: WorkerSourceFactory,
296 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
297 SourceState<S>: Send + 'static,
298 F: TransformFactory<S::Sample> + Send + Sync + 'static,
299 F::Transform: WorkerCheckpoint + Send + 'static,
300 TransformState<S, F>: Send + 'static,
301 TransformOutput<S, F>: Send + 'static,
302 TransformFailure<S, F>: Send + 'static,
303 F::Error: Send + 'static,
304{
305 fn new(
306 factory: Arc<S>,
307 transforms: Arc<F>,
308 config: StreamConfiguration,
309 source_generation: u64,
310 run: WorkerRunContext,
311 boundary: u64,
312 states: Option<Vec<LaneState<S, F>>>,
313 ) -> std::result::Result<Self, BuildError<S, F>> {
314 let journal_capacity = validate_capacity::<S, F>(config)?;
315 let (result_tx, results) = bounded_checked(
316 config.workers * config.prefetch_factor,
317 "exact stream results",
318 )?;
319 let (init_tx, initialized) =
320 bounded_checked(config.workers, "exact stream initialization")?;
321 let (ack_tx, acknowledgments) =
322 bounded_checked(config.workers, "exact stream acknowledgments")?;
323 let mut pool = Self {
324 commands: Vec::new(),
325 credits: Vec::new(),
326 results,
327 initialized,
328 acknowledgments,
329 handles: Vec::new(),
330 shutdown: CancellationToken::new(),
331 run,
332 committed: Arc::new(AtomicU64::new(boundary)),
333 };
334 let mut pending = states.map(Vec::into_iter);
335 for id in 0..config.workers {
336 let (command_tx, commands) = bounded(1);
337 let (credit_tx, credits) = bounded(config.prefetch_factor);
338 for _ in 0..config.prefetch_factor {
339 credit_tx.try_send(()).expect("fresh bounded credits");
340 }
341 pool.commands.push(command_tx);
342 pool.credits.push(credit_tx);
343 let source_factory = Arc::clone(&factory);
344 let transform_factory = Arc::clone(&transforms);
345 let run = pool.run.clone();
346 let shutdown = pool.shutdown.clone();
347 let committed = Arc::clone(&pool.committed);
348 let initialized = init_tx.clone();
349 let acknowledgments = ack_tx.clone();
350 let results = result_tx.clone();
351 let state = pending.as_mut().and_then(Iterator::next);
352 let info = WorkerInfo::from_loader_seed(
353 id,
354 config.workers,
355 config.loader_seed,
356 config.rank,
357 source_generation,
358 )?;
359 let mut journal = VecDeque::new();
360 journal
361 .try_reserve_exact(journal_capacity)
362 .map_err(|e| invalid(format!("snapshot journal unavailable: {e}")))?;
363 if journal.capacity() > journal_capacity {
364 return Err(
365 invalid("snapshot journal exceeds validated retention capacity").into(),
366 );
367 }
368 let handle = thread::Builder::new().name(format!("rusttorch-exact-stream-{id}")).spawn(move || {
369 let mut did_initialize = false;
370 let mut generation = run.generation;
371 let outcome = catch_unwind(AssertUnwindSafe(|| with_worker_info(info, || {
372 let context = WorkerContext::new(info, run.cancellation.clone(), run.deadline.clone());
373 let mut source = source_factory.create(context.clone()).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })?;
374 let mut transform = transform_factory.create(Some(&context)).map_err(|source| StreamCheckpointBuildError::TransformFactory { worker: id, source })?;
375 if let Some(state) = &state {
376 source.validate_snapshot(&state.source).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })?;
377 transform.validate_snapshot(&state.transform).map_err(|source| StreamCheckpointBuildError::TransformState { worker: id, source })?;
378 }
379 if initialized.send((id, Ok(()))).is_err() { return Ok(()); }
380 did_initialize = true;
381 let mut pending = state;
382 loop {
383 let command = select! {
384 recv(commands) -> command => match command { Ok(command) => command, Err(_) => return Ok(()) },
385 recv(shutdown.signal()) -> _ => return Ok(()),
386 };
387 match command {
388 Command::Apply => {
389 if let Some(state) = pending.take() {
390 source.restore_validated(&state.source);
391 transform.restore_validated(&state.transform);
392 }
393 }
394 Command::Begin(run) => {
395 generation = run.generation;
396 source.set_run_context(WorkerContext::new(info, run.cancellation.clone(), run.deadline.clone()));
397 let mut previous = None;
398 loop {
399 let acquired = select! {
400 recv(credits) -> result => result.is_ok(),
401 recv(run.cancellation.signal()) -> _ => false,
402 recv(shutdown.signal()) -> _ => false,
403 };
404 if !acquired || run.cancellation.is_cancelled() || shutdown.is_cancelled() { break; }
405 let boundary = committed.load(Ordering::Acquire);
406 while journal.front().is_some_and(|(attempt, _): &(Attempt, LaneState<S, F>)| attempt.sequence.is_some_and(|sequence| sequence < boundary)) { journal.pop_front(); }
407 assert!(journal.len() < journal_capacity, "validated stream snapshot window exhausted");
408 journal.push_back((Attempt { sequence: None, _logical_id: None, _outcome: AttemptOutcome::InFlight }, StreamLaneState { id, source: source.snapshot(), transform: transform.snapshot() }));
409 let event = match source.next() {
410 None => Event::End,
411 Some(Err(error)) => {
412 let sequence = source.error_sequence(&error).get();
413 if sequence < boundary || previous.is_some_and(|previous| sequence <= previous) {
414 Event::Protocol { sequence: Some(sequence), reason: "exact stream source error has a past or non-increasing sequence".to_owned() }
415 } else { Event::Source { sequence, error } }
416 },
417 Some(Ok(record)) => {
418 let sequence = record.sequence.map(SequenceId::get);
419 match sequence {
420 None => Event::Protocol { sequence, reason: "exact stream record is missing its sequence ID".to_owned() },
421 Some(sequence) if sequence < boundary || previous.is_some_and(|previous| sequence <= previous) => Event::Protocol { sequence: Some(sequence), reason: "exact stream shard emitted a past or non-increasing sequence".to_owned() },
422 Some(sequence) => {
423 previous = Some(sequence);
424 let logical_id = record.logical_id.get();
425 let task = TaskContext { loader_seed: run.loader_seed, epoch: run.epoch, rank: info.rank, logical_sample: logical_id, stage: 0, cancellation: run.cancellation.clone(), deadline: run.deadline.clone() };
426 match transform.transform(record.sample, &task) {
427 Ok(sample) => Event::Record(WorkerRecord { sequence: record.sequence, logical_id: record.logical_id, sample }),
428 Err(error) => Event::Transform { sequence, logical_id, error },
429 }
430 }
431 }
432 }
433 };
434 let (sequence, logical_id, outcome) = match &event {
435 Event::Record(record) => (record.sequence.map(SequenceId::get), Some(record.logical_id.get()), AttemptOutcome::Record),
436 Event::End => (None, None, AttemptOutcome::End),
437 Event::Source { sequence, .. } => (Some(*sequence), None, AttemptOutcome::SourceError),
438 Event::Transform { sequence, logical_id, .. } => (Some(*sequence), Some(*logical_id), AttemptOutcome::TransformError),
439 Event::Protocol { sequence, .. } => (*sequence, None, AttemptOutcome::Protocol),
440 Event::Panic => unreachable!("panic is handled outside production"),
441 };
442 journal.back_mut().expect("pre-attempt snapshot").0 = Attempt { sequence, _logical_id: logical_id, _outcome: outcome };
443 let terminal = !matches!(event, Event::Record(_));
444 let message = Completion { worker: id, generation, event };
445 let sent = select! {
446 send(results, message) -> result => result.is_ok(),
447 recv(run.cancellation.signal()) -> _ => false,
448 recv(shutdown.signal()) -> _ => false,
449 };
450 if terminal || !sent { break; }
451 }
452 }
453 Command::Rollback { generation: requested, boundary } => {
454 let state = if requested != generation { Err(invalid("stale rollback generation").into()) } else {
455 let state = journal.iter().find(|(attempt, _)| attempt.sequence.is_none_or(|sequence| sequence >= boundary)).map(|(_, state)| state.clone())
456 .unwrap_or_else(|| StreamLaneState { id, source: source.snapshot(), transform: transform.snapshot() });
457 source.validate_snapshot(&state.source).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })
458 .and_then(|()| transform.validate_snapshot(&state.transform).map_err(|source| StreamCheckpointBuildError::TransformState { worker: id, source }))
459 .map(|()| { source.restore_validated(&state.source); transform.restore_validated(&state.transform); state })
460 };
461 journal.clear();
462 while credits.try_recv().is_ok() {}
463 if acknowledgments.send((id, generation, state)).is_err() { return Ok(()); }
464 }
465 }
466 }
467 })));
468 match outcome {
469 Ok(Err(error)) => { let _ = initialized.send((id, Err(error))); }
470 Err(_) if !did_initialize => { let _ = initialized.send((id, Err(StreamCheckpointBuildError::WorkerPanic { worker: id }))); }
471 Err(_) => {
472 let message = Completion { worker: id, generation, event: Event::Panic };
473 select! { send(results, message) -> _ => {}, recv(shutdown.signal()) -> _ => {} }
474 }
475 Ok(Ok(())) => {}
476 }
477 }).map_err(|e| invalid(format!("cannot start exact stream worker: {e}")))?;
478 pool.handles.push(handle);
479 }
480 drop(init_tx);
481 drop(ack_tx);
482 drop(result_tx);
483 let mut seen = vec![false; config.workers];
484 for _ in 0..config.workers {
485 let (id, result) = pool
486 .initialized
487 .recv()
488 .map_err(|_| invalid("worker closed during validation"))?;
489 if id >= seen.len() || std::mem::replace(&mut seen[id], true) {
490 return Err(invalid("duplicate initialization lane").into());
491 }
492 result?;
493 }
494 Ok(pool)
495 }
496
497 fn begin(&self) -> Result<()> {
498 for sender in &self.commands {
499 sender
500 .send(Command::Begin(self.run.clone()))
501 .map_err(|_| invalid("worker closed before production"))?;
502 }
503 Ok(())
504 }
505
506 fn apply(&self) -> Result<()> {
507 for sender in &self.commands {
508 sender
509 .send(Command::Apply)
510 .map_err(|_| invalid("worker closed before validated apply"))?;
511 }
512 Ok(())
513 }
514
515 fn rollback(
516 &mut self,
517 boundary: u64,
518 prefetch: usize,
519 ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
520 let generation = self.run.generation;
521 let next_generation = generation
522 .checked_add(1)
523 .filter(|g| *g < u64::MAX)
524 .ok_or_else(|| invalid("transport generation overflow"))?;
525 self.run.cancel();
526 for command in &self.commands {
527 command
528 .send(Command::Rollback {
529 generation,
530 boundary,
531 })
532 .map_err(|_| invalid("worker failed before rollback"))?;
533 }
534 let states = (0..self.commands.len()).map(|_| None).collect::<Vec<_>>();
535 self.collect_rollback(states, generation, next_generation, boundary, prefetch)
536 }
537
538 fn collect_rollback(
539 &mut self,
540 mut states: Vec<Option<LaneState<S, F>>>,
541 generation: u64,
542 next_generation: u64,
543 boundary: u64,
544 prefetch: usize,
545 ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
546 while states.iter().any(Option::is_none) {
547 select! {
548 recv(self.acknowledgments) -> reply => {
549 let (id, received, state) = reply.map_err(|_| invalid("missing rollback acknowledgment"))?;
550 if id >= states.len() || received != generation || states[id].is_some() { return Err(invalid("stale or duplicate rollback acknowledgment").into()); }
551 states[id] = Some(state?);
552 },
553 recv(self.results) -> message => {
554 if matches!(message, Ok(Completion { event: Event::Panic, .. }) | Err(_)) { return Err(invalid("worker failed before rollback acknowledgment").into()); }
555 },
556 }
557 }
558 self.finish_rollback(states, next_generation, boundary, prefetch)
559 }
560
561 fn finish_rollback(
562 &mut self,
563 states: Vec<Option<LaneState<S, F>>>,
564 generation: u64,
565 boundary: u64,
566 prefetch: usize,
567 ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
568 if self.acknowledgments.try_recv().is_ok() {
569 return Err(invalid("duplicate rollback acknowledgment").into());
570 }
571 while self.results.try_recv().is_ok() {}
572 for credit in &self.credits {
573 for _ in 0..prefetch {
574 credit
575 .try_send(())
576 .map_err(|_| invalid("rollback credits were not drained"))?;
577 }
578 }
579 self.committed.store(boundary, Ordering::Release);
580 self.run = WorkerRunContext::new(generation, self.run.loader_seed, self.run.epoch);
581 Ok(states.into_iter().map(Option::unwrap).collect())
582 }
583}
584
585pub struct ExactStreamDataLoader<S, C, F = IdentityTransformFactory, N = PinDisabled>
692where
693 S: WorkerSourceFactory,
694 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
695 F: TransformFactory<S::Sample>,
696 F::Transform: WorkerCheckpoint,
697 C: Collate<TransformOutput<S, F>> + Checkpointable,
698{
699 factory: Arc<S>,
700 transform_factory: Arc<F>,
701 collator: C,
702 configuration: StreamConfiguration,
703 identity: String,
704 pin: PinMemoryStatus,
705 length: Option<usize>,
706 pending: Option<Pool<S, F>>,
707 next_batch: u64,
708 next_sequence: u64,
709 short_tail: bool,
710 source_generation: u64,
711 next_generation: u64,
712 policies: PhantomData<N>,
713}
714
715impl<S, C, F, N> StreamDataLoaderBuilder<S, C, F, NoWorkerInit, MemoryDisabled, N, CheckpointFresh>
716where
717 S: CheckpointSourceFactory,
718 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
719 SourceState<S>: Send + 'static,
720 F: TransformFactory<S::Sample> + Send + Sync + 'static,
721 F::Transform: WorkerCheckpoint + Send + 'static,
722 TransformState<S, F>: Send + 'static,
723 TransformOutput<S, F>: Send + 'static,
724 TransformFailure<S, F>: Send + 'static,
725 F::Error: Send + 'static,
726 C: Collate<TransformOutput<S, F>> + Checkpointable,
727 N: StreamPinPolicy<S, C, F>,
728{
729 pub fn build(self) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>> {
731 let identity = self.checkpoint.identity.clone();
732 build(self, identity, None)
733 }
734}
735
736impl<S, C, F, N>
737 StreamDataLoaderBuilder<
738 S,
739 C,
740 F,
741 NoWorkerInit,
742 MemoryDisabled,
743 N,
744 CheckpointResume<State<S, F, C>>,
745 >
746where
747 S: CheckpointSourceFactory,
748 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
749 SourceState<S>: Send + 'static,
750 F: TransformFactory<S::Sample> + Send + Sync + 'static,
751 F::Transform: WorkerCheckpoint + Send + 'static,
752 TransformState<S, F>: Send + 'static,
753 TransformOutput<S, F>: Send + 'static,
754 TransformFailure<S, F>: Send + 'static,
755 F::Error: Send + 'static,
756 C: Collate<TransformOutput<S, F>> + Checkpointable,
757 N: StreamPinPolicy<S, C, F>,
758{
759 pub fn build(self) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>> {
761 let StreamDataLoaderBuilder {
762 factory,
763 collator,
764 transform_factory,
765 worker_init,
766 configuration,
767 checkpoint,
768 ..
769 } = self;
770 let builder = StreamDataLoaderBuilder {
771 factory,
772 collator,
773 transform_factory,
774 worker_init,
775 configuration,
776 checkpoint: CheckpointDisabled,
777 states: PhantomData,
778 };
779 build(builder, checkpoint.identity, Some(checkpoint.state))
780 }
781}
782
783fn build<S, C, F, N, K>(
784 builder: StreamDataLoaderBuilder<S, C, F, NoWorkerInit, MemoryDisabled, N, K>,
785 identity: String,
786 state: Option<State<S, F, C>>,
787) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>>
788where
789 S: CheckpointSourceFactory,
790 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
791 SourceState<S>: Send + 'static,
792 F: TransformFactory<S::Sample> + Send + Sync + 'static,
793 F::Transform: WorkerCheckpoint + Send + 'static,
794 TransformState<S, F>: Send + 'static,
795 TransformOutput<S, F>: Send + 'static,
796 TransformFailure<S, F>: Send + 'static,
797 F::Error: Send + 'static,
798 C: Collate<TransformOutput<S, F>> + Checkpointable,
799 N: StreamPinPolicy<S, C, F>,
800{
801 let config = builder.configuration;
802 validate_capacity::<S, F>(config)?;
803 if identity.is_empty() || S::CHECKPOINT_KIND.is_empty() {
804 return Err(invalid("source identity and factory kind must not be empty").into());
805 }
806 let pin = resolve_stream_pin(config.pin_memory)?;
807 if let Some(state) = &state {
808 let batch_size = u64::try_from(config.batch_size)
809 .map_err(|_| invalid("batch size exceeds sequence range"))?;
810 let full = state
811 .next_batch
812 .checked_mul(batch_size)
813 .ok_or_else(|| invalid("batch cursor overflow"))?;
814 let short_tail = !config.drop_last
815 && state.next_batch > 0
816 && state.next_sequence < full
817 && state.next_sequence > full - batch_size;
818 if state.schema_version != 1
819 || state.source_identity != identity
820 || state.factory_kind != S::CHECKPOINT_KIND
821 || state.epoch != config.epoch
822 || state.source_generation > state.transport_generation
823 || state.transport_generation >= u64::MAX - 1
824 || state.rng_derivation_version != TASK_RNG_DERIVATION_VERSION
825 || state.worker_seed_derivation_version != WORKER_SEED_DERIVATION_VERSION
826 || state.configuration != configuration(config, pin)
827 || state.lanes.len() != config.workers
828 || state
829 .lanes
830 .iter()
831 .enumerate()
832 .any(|(id, lane)| lane.id != id)
833 || state.short_tail != short_tail
834 || (state.next_sequence != full && !short_tail)
835 || state.source_length.is_some_and(|length| {
836 state.next_sequence > length as u64
837 || (short_tail && state.next_sequence != length as u64)
838 })
839 {
840 return Err(invalid("incompatible or corrupt stream checkpoint envelope").into());
841 }
842 }
843 let length = builder.factory.exact_len();
844 if state
845 .as_ref()
846 .is_some_and(|state| state.source_length != length)
847 {
848 return Err(invalid("source length changed").into());
849 }
850 let mut loader = ExactStreamDataLoader {
851 factory: Arc::new(builder.factory),
852 transform_factory: Arc::new(builder.transform_factory),
853 collator: builder.collator,
854 configuration: config,
855 identity,
856 pin,
857 length,
858 pending: None,
859 next_batch: state.as_ref().map_or(0, |s| s.next_batch),
860 next_sequence: state.as_ref().map_or(0, |s| s.next_sequence),
861 short_tail: state.as_ref().is_some_and(|s| s.short_tail),
862 source_generation: state.as_ref().map_or(0, |s| s.source_generation),
863 next_generation: state.as_ref().map_or(1, |s| s.transport_generation + 1),
864 policies: PhantomData,
865 };
866 let generation = state.as_ref().map_or(0, |s| s.transport_generation);
867 let pool = Pool::new(
868 Arc::clone(&loader.factory),
869 Arc::clone(&loader.transform_factory),
870 config,
871 loader.source_generation,
872 WorkerRunContext::new(generation, config.loader_seed, config.epoch),
873 loader.next_sequence,
874 state.as_ref().map(|s| s.lanes.clone()),
875 )?;
876 if let Some(state) = &state {
877 loader.collator.validate_state(&state.coordinator)?;
878 pool.apply()?;
879 loader.collator.load_validated(&state.coordinator);
880 }
881 loader.pending = Some(pool);
882 Ok(loader)
883}
884
885pub struct ExactStreamLoaderIter<'a, S, C, F = IdentityTransformFactory, N = PinDisabled>
892where
893 S: WorkerSourceFactory,
894 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
895 F: TransformFactory<S::Sample>,
896 F::Transform: WorkerCheckpoint,
897 C: Collate<TransformOutput<S, F>> + Checkpointable,
898{
899 loader: &'a mut ExactStreamDataLoader<S, C, F, N>,
900 pool: Option<Pool<S, F>>,
901 completed: Vec<Buffered<S, F>>,
902 ended: Vec<bool>,
903 next_sequence: u64,
904 visible_sequence: u64,
905 short_tail: bool,
906 next_batch: u64,
907 stopped: bool,
908 boundary_valid: bool,
909 pending_error: Option<IterError<S, F, C>>,
910}
911
912impl<S, C, F, N> ExactStreamDataLoader<S, C, F, N>
913where
914 S: CheckpointSourceFactory,
915 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
916 SourceState<S>: Send + 'static,
917 F: TransformFactory<S::Sample> + Send + Sync + 'static,
918 F::Transform: WorkerCheckpoint + Send + 'static,
919 TransformState<S, F>: Send + 'static,
920 TransformOutput<S, F>: Send + 'static,
921 TransformFailure<S, F>: Send + 'static,
922 F::Error: Send + 'static,
923 C: Collate<TransformOutput<S, F>> + Checkpointable,
924 N: StreamPinPolicy<S, C, F>,
925{
926 pub fn len(&self) -> Option<usize> {
928 self.length.map(|n| {
929 if self.configuration.drop_last {
930 n / self.configuration.batch_size
931 } else {
932 n.div_ceil(self.configuration.batch_size)
933 }
934 })
935 }
936 pub fn is_empty(&self) -> Option<bool> {
938 self.len().map(|n| n == 0)
939 }
940 pub fn workers(&self) -> usize {
942 self.configuration.workers
943 }
944 pub fn pin_memory_status(&self) -> PinMemoryStatus {
946 self.pin
947 }
948 pub fn set_epoch(&mut self, epoch: u64) {
950 self.pending = None;
951 self.configuration.epoch = epoch;
952 }
953 pub fn iter(&mut self) -> ExactStreamLoaderIter<'_, S, C, F, N> {
959 let retained = self.pending.is_some();
960 let mut pending_error = None;
961 let pool = if retained {
962 self.pending.take()
963 } else {
964 let generation = self.next_generation;
965 if let Some(next) = generation.checked_add(1).filter(|g| *g < u64::MAX) {
966 self.next_generation = next;
967 self.source_generation = generation;
968 match Pool::new(
969 Arc::clone(&self.factory),
970 Arc::clone(&self.transform_factory),
971 self.configuration,
972 generation,
973 WorkerRunContext::new(
974 generation,
975 self.configuration.loader_seed,
976 self.configuration.epoch,
977 ),
978 0,
979 None,
980 ) {
981 Ok(pool) => Some(pool),
982 Err(error) => {
983 pending_error = Some(map_build_error::<S, F, C>(error));
984 None
985 }
986 }
987 } else {
988 pending_error = Some(LoaderError::Configuration(invalid("generation overflow")));
989 None
990 }
991 };
992 if let Some(pool) = &pool
993 && let Err(error) = pool.begin()
994 {
995 pending_error = Some(LoaderError::Configuration(error));
996 }
997 let next_batch = if retained { self.next_batch } else { 0 };
998 let next_sequence = if retained { self.next_sequence } else { 0 };
999 let short_tail = retained && self.short_tail;
1000 let ended = vec![false; self.configuration.workers];
1001 let completed =
1002 Vec::with_capacity(self.configuration.workers * self.configuration.prefetch_factor);
1003 ExactStreamLoaderIter {
1004 loader: self,
1005 pool,
1006 completed,
1007 ended,
1008 next_batch,
1009 next_sequence,
1010 visible_sequence: next_sequence,
1011 short_tail,
1012 stopped: false,
1013 boundary_valid: pending_error.is_none(),
1014 pending_error,
1015 }
1016 }
1017}
1018
1019fn map_build_error<S, F, C>(error: BuildError<S, F>) -> IterError<S, F, C>
1020where
1021 S: WorkerSourceFactory,
1022 F: TransformFactory<S::Sample>,
1023 C: Collate<TransformOutput<S, F>>,
1024{
1025 match error {
1026 StreamCheckpointBuildError::Configuration(e) => LoaderError::Configuration(e),
1027 StreamCheckpointBuildError::Source { worker, source } => LoaderError::StreamPipeline {
1028 batch: None,
1029 worker,
1030 sequence: None,
1031 logical_id: None,
1032 source: PipelineError::Source(source),
1033 },
1034 StreamCheckpointBuildError::TransformFactory { worker, source } => {
1035 LoaderError::StreamPipeline {
1036 batch: None,
1037 worker,
1038 sequence: None,
1039 logical_id: None,
1040 source: PipelineError::TransformInit(source),
1041 }
1042 }
1043 StreamCheckpointBuildError::TransformState { source, .. } => {
1044 LoaderError::Configuration(source)
1045 }
1046 StreamCheckpointBuildError::WorkerPanic { worker } => LoaderError::WorkerPanic {
1047 worker,
1048 batch: None,
1049 },
1050 }
1051}
1052
1053impl<S, C, F, N> ExactStreamLoaderIter<'_, S, C, F, N>
1054where
1055 S: CheckpointSourceFactory,
1056 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
1057 SourceState<S>: Send + 'static,
1058 F: TransformFactory<S::Sample> + Send + Sync + 'static,
1059 F::Transform: WorkerCheckpoint + Send + 'static,
1060 TransformState<S, F>: Send + 'static,
1061 TransformOutput<S, F>: Send + 'static,
1062 TransformFailure<S, F>: Send + 'static,
1063 F::Error: Send + 'static,
1064 C: Collate<TransformOutput<S, F>> + Checkpointable,
1065 N: StreamPinPolicy<S, C, F>,
1066{
1067 pub fn checkpoint(&mut self) -> std::result::Result<State<S, F, C>, IterError<S, F, C>> {
1080 if !self.boundary_valid {
1081 return Err(LoaderError::Checkpoint {
1082 reason: "no recoverable visible stream boundary".to_owned(),
1083 });
1084 }
1085 let coordinator = self.loader.collator.save_state();
1086 let pool = self.pool.as_mut().ok_or_else(|| LoaderError::Checkpoint {
1087 reason: "missing worker pool".to_owned(),
1088 })?;
1089 let lanes = match pool.rollback(
1090 self.visible_sequence,
1091 self.loader.configuration.prefetch_factor,
1092 ) {
1093 Ok(lanes) => lanes,
1094 Err(error) => {
1095 self.boundary_valid = false;
1096 self.stopped = true;
1097 self.pool = None;
1098 return Err(map_build_error::<S, F, C>(error));
1099 }
1100 };
1101 self.completed.clear();
1102 self.ended.fill(false);
1103 self.next_sequence = self.visible_sequence;
1104 self.stopped = false;
1105 let state = StreamLoaderState {
1106 schema_version: 1,
1107 source_identity: self.loader.identity.clone(),
1108 factory_kind: S::CHECKPOINT_KIND.to_owned(),
1109 epoch: self.loader.configuration.epoch,
1110 source_generation: self.loader.source_generation,
1111 transport_generation: pool.run.generation,
1112 next_batch: self.next_batch,
1113 next_sequence: self.visible_sequence,
1114 short_tail: self.short_tail,
1115 source_length: self.loader.length,
1116 rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
1117 worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
1118 configuration: configuration(self.loader.configuration, self.loader.pin),
1119 lanes,
1120 coordinator,
1121 };
1122 self.loader.next_generation = self.loader.next_generation.max(pool.run.generation + 1);
1123 pool.begin().map_err(LoaderError::Configuration)?;
1124 Ok(state)
1125 }
1126
1127 fn fail(
1128 &mut self,
1129 error: IterError<S, F, C>,
1130 recoverable: bool,
1131 ) -> Option<std::result::Result<C::Batch, IterError<S, F, C>>> {
1132 self.stopped = true;
1133 self.boundary_valid &= recoverable;
1134 if let Some(pool) = &self.pool {
1135 pool.run.cancel();
1136 }
1137 if !recoverable {
1138 self.pool = None;
1139 }
1140 Some(Err(error))
1141 }
1142}
1143
1144impl<S, C, F, N> Iterator for ExactStreamLoaderIter<'_, S, C, F, N>
1145where
1146 S: CheckpointSourceFactory,
1147 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
1148 SourceState<S>: Send + 'static,
1149 F: TransformFactory<S::Sample> + Send + Sync + 'static,
1150 F::Transform: WorkerCheckpoint + Send + 'static,
1151 TransformState<S, F>: Send + 'static,
1152 TransformOutput<S, F>: Send + 'static,
1153 TransformFailure<S, F>: Send + 'static,
1154 F::Error: Send + 'static,
1155 C: Collate<TransformOutput<S, F>> + Checkpointable,
1156 N: StreamPinPolicy<S, C, F>,
1157{
1158 type Item = std::result::Result<C::Batch, IterError<S, F, C>>;
1159 fn next(&mut self) -> Option<Self::Item> {
1160 if self.stopped {
1161 return None;
1162 }
1163 if let Some(error) = self.pending_error.take() {
1164 return self.fail(error, false);
1165 }
1166 let mut samples = Vec::with_capacity(self.loader.configuration.batch_size);
1167 loop {
1168 if let Some(index) = self
1169 .completed
1170 .iter()
1171 .position(|(_, event)| event.sequence() == Some(self.next_sequence))
1172 {
1173 let (worker, event) = self.completed.swap_remove(index);
1174 let record = match event {
1175 Event::Record(record) => record,
1176 Event::Source { sequence, error } => {
1177 return self.fail(
1178 LoaderError::StreamPipeline {
1179 batch: Some(self.next_batch),
1180 worker,
1181 sequence: Some(sequence),
1182 logical_id: None,
1183 source: PipelineError::Source(error),
1184 },
1185 true,
1186 );
1187 }
1188 Event::Transform {
1189 sequence,
1190 logical_id,
1191 error,
1192 } => {
1193 return self.fail(
1194 LoaderError::StreamPipeline {
1195 batch: Some(self.next_batch),
1196 worker,
1197 sequence: Some(sequence),
1198 logical_id: Some(logical_id),
1199 source: PipelineError::Transform(error),
1200 },
1201 true,
1202 );
1203 }
1204 _ => unreachable!("only sequenced outcomes enter reassembly"),
1205 };
1206 let Some(next) = self.next_sequence.checked_add(1) else {
1207 return self.fail(
1208 LoaderError::StreamProtocol {
1209 sequence: Some(self.next_sequence),
1210 reason: "sequence overflow".to_owned(),
1211 },
1212 false,
1213 );
1214 };
1215 self.next_sequence = next;
1216 samples.push(record.sample);
1217 if self.pool.as_ref().expect("active pool").credits[worker]
1218 .try_send(())
1219 .is_err()
1220 {
1221 return self.fail(
1222 LoaderError::ChannelClosed {
1223 batch: self.next_batch,
1224 },
1225 false,
1226 );
1227 }
1228 if samples.len() == self.loader.configuration.batch_size {
1229 break;
1230 }
1231 continue;
1232 }
1233 if self.ended.iter().all(|ended| *ended) {
1234 if !self.completed.is_empty()
1235 || self
1236 .loader
1237 .length
1238 .is_some_and(|length| self.next_sequence != length as u64)
1239 {
1240 return self.fail(
1241 LoaderError::StreamProtocol {
1242 sequence: Some(self.next_sequence),
1243 reason:
1244 "stream ended with a missing global sequence or wrong exact length"
1245 .to_owned(),
1246 },
1247 false,
1248 );
1249 }
1250 if samples.is_empty() || self.loader.configuration.drop_last {
1251 self.stopped = true;
1252 return None;
1253 }
1254 break;
1255 }
1256 if self.ended.iter().enumerate().all(|(id, ended)| {
1258 *ended
1259 || self
1260 .completed
1261 .iter()
1262 .filter(|(worker, _)| *worker == id)
1263 .count()
1264 == self.loader.configuration.prefetch_factor
1265 }) {
1266 return self.fail(
1267 LoaderError::StreamProtocol {
1268 sequence: Some(self.next_sequence),
1269 reason: "bounded stream window contains no expected sequence".to_owned(),
1270 },
1271 false,
1272 );
1273 }
1274 let pool = self.pool.as_ref().expect("active pool");
1275 let completion = match pool.results.recv() {
1276 Ok(completion) => completion,
1277 Err(_) => {
1278 return self.fail(
1279 LoaderError::ChannelClosed {
1280 batch: self.next_batch,
1281 },
1282 false,
1283 );
1284 }
1285 };
1286 if completion.generation != pool.run.generation {
1287 return self.fail(
1288 LoaderError::StreamProtocol {
1289 sequence: None,
1290 reason: "stale transport message".to_owned(),
1291 },
1292 false,
1293 );
1294 }
1295 let worker = completion.worker;
1296 match completion.event {
1297 event @ (Event::Record(_) | Event::Source { .. } | Event::Transform { .. }) => {
1298 let sequence = event.sequence();
1299 if sequence.is_none_or(|sequence| sequence < self.next_sequence)
1300 || self
1301 .completed
1302 .iter()
1303 .any(|(_, previous)| previous.sequence() == sequence)
1304 {
1305 return self.fail(
1306 LoaderError::StreamProtocol {
1307 sequence,
1308 reason: "missing, duplicate, or past sequence ID".to_owned(),
1309 },
1310 false,
1311 );
1312 }
1313 if !matches!(event, Event::Record(_)) {
1314 self.ended[worker] = true;
1315 }
1316 self.completed.push((worker, event));
1317 }
1318 Event::End => {
1319 self.ended[worker] = true;
1320 }
1321 Event::Protocol { sequence, reason } => {
1322 return self.fail(LoaderError::StreamProtocol { sequence, reason }, false);
1323 }
1324 Event::Panic => {
1325 return self.fail(
1326 LoaderError::StreamWorkerPanic {
1327 batch: Some(self.next_batch),
1328 worker,
1329 sequence: None,
1330 logical_id: None,
1331 },
1332 false,
1333 );
1334 }
1335 }
1336 }
1337 let short_tail = samples.len() < self.loader.configuration.batch_size;
1338 let batch = match catch_unwind(AssertUnwindSafe(|| self.loader.collator.collate(samples))) {
1339 Ok(Ok(batch)) => batch,
1340 Ok(Err(source)) => {
1341 return self.fail(
1342 LoaderError::Pipeline {
1343 batch: Some(self.next_batch),
1344 worker: None,
1345 source: PipelineError::Collate(source),
1346 },
1347 false,
1348 );
1349 }
1350 Err(_) => {
1351 return self.fail(
1352 LoaderError::CoordinatorPanic {
1353 stage: "collation",
1354 batch: Some(self.next_batch),
1355 },
1356 false,
1357 );
1358 }
1359 };
1360 let batch = if let PinMemoryStatus::Enabled(device) = self.loader.pin {
1361 match catch_unwind(AssertUnwindSafe(|| N::pin(batch, device))) {
1362 Ok(Ok(batch)) => batch,
1363 Ok(Err(source)) => {
1364 return self.fail(
1365 LoaderError::PinMemory {
1366 batch: self.next_batch,
1367 source,
1368 },
1369 false,
1370 );
1371 }
1372 Err(_) => {
1373 return self.fail(
1374 LoaderError::CoordinatorPanic {
1375 stage: "pinning",
1376 batch: Some(self.next_batch),
1377 },
1378 false,
1379 );
1380 }
1381 }
1382 } else {
1383 batch
1384 };
1385 let Some(next_batch) = self.next_batch.checked_add(1) else {
1386 return self.fail(
1387 LoaderError::Configuration(invalid("batch cursor overflow")),
1388 false,
1389 );
1390 };
1391 self.next_batch = next_batch;
1392 self.visible_sequence = self.next_sequence;
1393 self.short_tail = short_tail;
1394 self.pool
1395 .as_ref()
1396 .expect("active pool")
1397 .committed
1398 .store(self.visible_sequence, Ordering::Release);
1399 Some(Ok(batch))
1400 }
1401}