1use super::*;
4use crate::{
5 CheckpointDisabled, CheckpointFresh, CheckpointResume, CheckpointSourceFactory, Checkpointable,
6 CheckpointableSource, StreamCheckpointBuildError, StreamCheckpointConfiguration,
7 StreamLaneState, StreamLoaderState, TASK_RNG_DERIVATION_VERSION,
8 WORKER_SEED_DERIVATION_VERSION, WorkerCheckpoint,
9};
10use std::collections::VecDeque;
11use std::sync::atomic::{AtomicU64, Ordering};
12
13type SourceState<S> = <<S as WorkerSourceFactory>::Source as CheckpointableSource>::State;
14type TransformState<S, F> = <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as WorkerCheckpoint>::State;
15type LaneState<S, F> = StreamLaneState<SourceState<S>, TransformState<S, F>>;
16type State<S, F, C> =
17 StreamLoaderState<SourceState<S>, TransformState<S, F>, <C as Checkpointable>::State>;
18type BuildError<S, F> = StreamCheckpointBuildError<
19 <S as WorkerSourceFactory>::Error,
20 <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
21>;
22type IterError<S, F, C> = StreamLoaderError<S, C, F, NoWorkerInit>;
23
24fn invalid(reason: impl Into<String>) -> RustTorchError {
25 invalid_configuration("stream_checkpoint", reason)
26}
27
28impl<S, C, F, I, M, N> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointDisabled> {
29 pub fn checkpointable(
41 self,
42 identity: impl Into<String>,
43 ) -> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointFresh> {
44 StreamDataLoaderBuilder {
45 factory: self.factory,
46 collator: self.collator,
47 transform_factory: self.transform_factory,
48 worker_init: self.worker_init,
49 configuration: self.configuration,
50 states: PhantomData,
51 checkpoint: CheckpointFresh {
52 identity: identity.into(),
53 },
54 }
55 }
56
57 #[allow(clippy::type_complexity)]
59 pub fn resume<SS, TS, CS>(
60 self,
61 identity: impl Into<String>,
62 state: StreamLoaderState<SS, TS, CS>,
63 ) -> StreamDataLoaderBuilder<S, C, F, I, M, N, CheckpointResume<StreamLoaderState<SS, TS, CS>>>
64 {
65 StreamDataLoaderBuilder {
66 factory: self.factory,
67 collator: self.collator,
68 transform_factory: self.transform_factory,
69 worker_init: self.worker_init,
70 configuration: self.configuration,
71 states: PhantomData,
72 checkpoint: CheckpointResume {
73 identity: identity.into(),
74 state,
75 },
76 }
77 }
78}
79
80fn configuration(
81 config: StreamConfiguration,
82 pin: PinMemoryStatus,
83) -> StreamCheckpointConfiguration {
84 StreamCheckpointConfiguration {
85 workers: config.workers,
86 prefetch_factor: config.prefetch_factor,
87 batch_size: config.batch_size,
88 drop_last: config.drop_last,
89 loader_seed: config.loader_seed,
90 rank: config.rank,
91 pin_request: match config.pin_memory {
92 StreamPinRequest::Disabled => crate::CheckpointPinRequest::Disabled,
93 StreamPinRequest::Auto => crate::CheckpointPinRequest::Auto,
94 StreamPinRequest::Explicit(Device::Cuda(index)) => {
95 crate::CheckpointPinRequest::ExplicitCuda { index }
96 }
97 _ => unreachable!("pin request validated"),
98 },
99 pin_status: match pin {
100 PinMemoryStatus::Disabled => crate::CheckpointPinStatus::Disabled,
101 PinMemoryStatus::DisabledNoAccelerator => {
102 crate::CheckpointPinStatus::DisabledNoAccelerator
103 }
104 PinMemoryStatus::Enabled(Device::Cuda(index)) => {
105 crate::CheckpointPinStatus::Cuda { index }
106 }
107 _ => unreachable!("pin status validated"),
108 },
109 }
110}
111
112enum Command {
113 Apply,
114 Begin(WorkerRunContext),
115 Rollback { generation: u64, boundary: u64 },
116}
117
118enum Event<T, SE, TE> {
119 Record(WorkerRecord<T>),
120 End,
121 Source {
122 sequence: u64,
123 error: SE,
124 },
125 Transform {
126 sequence: u64,
127 logical_id: u64,
128 error: TE,
129 },
130 Protocol {
131 sequence: Option<u64>,
132 reason: String,
133 },
134 Panic,
135}
136
137impl<T, SE, TE> Event<T, SE, TE> {
138 fn sequence(&self) -> Option<u64> {
139 match self {
140 Self::Record(record) => record.sequence.map(SequenceId::get),
141 Self::Source { sequence, .. } | Self::Transform { sequence, .. } => Some(*sequence),
142 _ => None,
143 }
144 }
145}
146
147type Buffered<S, F> = (
148 usize,
149 Event<TransformOutput<S, F>, <S as WorkerSourceFactory>::Error, TransformFailure<S, F>>,
150);
151
152enum AttemptOutcome {
153 InFlight,
154 Record,
155 End,
156 SourceError,
157 TransformError,
158 Protocol,
159}
160
161struct Attempt {
162 sequence: Option<u64>,
163 _logical_id: Option<u64>,
164 _outcome: AttemptOutcome,
165}
166
167struct Completion<T, SE, TE> {
168 worker: usize,
169 generation: u64,
170 event: Event<T, SE, TE>,
171}
172type CompletionFor<S, F> =
173 Completion<TransformOutput<S, F>, <S as WorkerSourceFactory>::Error, TransformFailure<S, F>>;
174type Ack<S, F> = (
175 usize,
176 u64,
177 std::result::Result<LaneState<S, F>, BuildError<S, F>>,
178);
179
180struct Pool<S, F>
181where
182 S: WorkerSourceFactory,
183 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
184 F: TransformFactory<S::Sample>,
185 F::Transform: WorkerCheckpoint,
186{
187 commands: Vec<Sender<Command>>,
188 credits: Vec<Sender<()>>,
189 results: Receiver<CompletionFor<S, F>>,
190 #[allow(clippy::type_complexity)]
191 initialized: Receiver<(usize, std::result::Result<(), BuildError<S, F>>)>,
192 acknowledgments: Receiver<Ack<S, F>>,
193 handles: Vec<JoinHandle<()>>,
194 shutdown: CancellationToken,
195 run: WorkerRunContext,
196 committed: Arc<AtomicU64>,
197}
198
199impl<S, F> Drop for Pool<S, F>
200where
201 S: WorkerSourceFactory,
202 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
203 F: TransformFactory<S::Sample>,
204 F::Transform: WorkerCheckpoint,
205{
206 fn drop(&mut self) {
207 self.run.cancel();
208 self.shutdown.cancel();
209 for handle in self.handles.drain(..) {
210 let _ = handle.join();
211 }
212 }
213}
214
215fn validate_capacity<S, F>(config: StreamConfiguration) -> Result<usize>
216where
217 S: WorkerSourceFactory,
218 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
219 F: TransformFactory<S::Sample>,
220 F::Transform: WorkerCheckpoint,
221{
222 if config.workers == 0 || config.prefetch_factor == 0 || config.batch_size == 0 {
223 return Err(invalid(
224 "workers, prefetch_factor, and batch_size must be positive",
225 ));
226 }
227 if !config.ordered
228 || config.persistent_workers
229 || config.timeout.is_some()
230 || config.prefetch_bytes.is_some()
231 {
232 return Err(invalid(
233 "exact streams require ordered nonpersistent workers without timeout or byte accounting",
234 ));
235 }
236 let outstanding = config
237 .workers
238 .checked_mul(config.prefetch_factor)
239 .ok_or_else(|| invalid("outstanding overflow"))?;
240 let journal = config
243 .prefetch_factor
244 .checked_add(config.batch_size)
245 .and_then(|n| n.checked_add(1))
246 .ok_or_else(|| invalid("journal window overflow"))?;
247 let extra = config
248 .workers
249 .checked_mul(journal)
250 .and_then(|n| n.checked_mul(size_of::<(Attempt, LaneState<S, F>)>()))
251 .and_then(|n| {
252 n.checked_add(
253 config.workers.checked_mul(
254 size_of::<LaneState<S, F>>()
255 .checked_mul(8)?
256 .checked_add(8192)?
257 .checked_add(size_of::<CapacitySlot<Ack<S, F>>>())?
258 .checked_add(size_of::<
259 CapacitySlot<(usize, std::result::Result<(), BuildError<S, F>>)>,
260 >())?,
261 )?,
262 )
263 })
264 .and_then(|n| {
265 n.checked_add(outstanding.checked_mul(size_of::<CapacitySlot<CompletionFor<S, F>>>())?)
266 })
267 .and_then(|n| n.checked_add(outstanding.checked_mul(size_of::<Buffered<S, F>>())?))
268 .ok_or_else(|| invalid("snapshot storage overflow"))?;
269 validate_stream_capacity_extra::<S, F, NoWorkerInit, MemoryDisabled>(
270 config.workers,
271 config.prefetch_factor,
272 outstanding,
273 config.batch_size,
274 outstanding,
275 extra,
276 )?;
277 preflight_vec::<(Attempt, LaneState<S, F>)>(journal, "stream checkpoint journal")?;
278 Ok(journal)
279}
280
281impl<S, F> Pool<S, F>
282where
283 S: WorkerSourceFactory,
284 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
285 SourceState<S>: Send + 'static,
286 F: TransformFactory<S::Sample> + Send + Sync + 'static,
287 F::Transform: WorkerCheckpoint + Send + 'static,
288 TransformState<S, F>: Send + 'static,
289 TransformOutput<S, F>: Send + 'static,
290 TransformFailure<S, F>: Send + 'static,
291 F::Error: Send + 'static,
292{
293 fn new(
294 factory: Arc<S>,
295 transforms: Arc<F>,
296 config: StreamConfiguration,
297 source_generation: u64,
298 run: WorkerRunContext,
299 boundary: u64,
300 states: Option<Vec<LaneState<S, F>>>,
301 ) -> std::result::Result<Self, BuildError<S, F>> {
302 let journal_capacity = validate_capacity::<S, F>(config)?;
303 let (result_tx, results) = bounded_checked(
304 config.workers * config.prefetch_factor,
305 "exact stream results",
306 )?;
307 let (init_tx, initialized) =
308 bounded_checked(config.workers, "exact stream initialization")?;
309 let (ack_tx, acknowledgments) =
310 bounded_checked(config.workers, "exact stream acknowledgments")?;
311 let mut pool = Self {
312 commands: Vec::new(),
313 credits: Vec::new(),
314 results,
315 initialized,
316 acknowledgments,
317 handles: Vec::new(),
318 shutdown: CancellationToken::new(),
319 run,
320 committed: Arc::new(AtomicU64::new(boundary)),
321 };
322 let mut pending = states.map(Vec::into_iter);
323 for id in 0..config.workers {
324 let (command_tx, commands) = bounded(1);
325 let (credit_tx, credits) = bounded(config.prefetch_factor);
326 for _ in 0..config.prefetch_factor {
327 credit_tx.try_send(()).expect("fresh bounded credits");
328 }
329 pool.commands.push(command_tx);
330 pool.credits.push(credit_tx);
331 let source_factory = Arc::clone(&factory);
332 let transform_factory = Arc::clone(&transforms);
333 let run = pool.run.clone();
334 let shutdown = pool.shutdown.clone();
335 let committed = Arc::clone(&pool.committed);
336 let initialized = init_tx.clone();
337 let acknowledgments = ack_tx.clone();
338 let results = result_tx.clone();
339 let state = pending.as_mut().and_then(Iterator::next);
340 let info = WorkerInfo::from_loader_seed(
341 id,
342 config.workers,
343 config.loader_seed,
344 config.rank,
345 source_generation,
346 )?;
347 let mut journal = VecDeque::new();
348 journal
349 .try_reserve_exact(journal_capacity)
350 .map_err(|e| invalid(format!("snapshot journal unavailable: {e}")))?;
351 if journal.capacity() > journal_capacity {
352 return Err(
353 invalid("snapshot journal exceeds validated retention capacity").into(),
354 );
355 }
356 let handle = thread::Builder::new().name(format!("rusttorch-exact-stream-{id}")).spawn(move || {
357 let mut did_initialize = false;
358 let mut generation = run.generation;
359 let outcome = catch_unwind(AssertUnwindSafe(|| with_worker_info(info, || {
360 let context = WorkerContext::new(info, run.cancellation.clone(), run.deadline.clone());
361 let mut source = source_factory.create(context.clone()).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })?;
362 let mut transform = transform_factory.create(Some(&context)).map_err(|source| StreamCheckpointBuildError::TransformFactory { worker: id, source })?;
363 if let Some(state) = &state {
364 source.validate_snapshot(&state.source).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })?;
365 transform.validate_snapshot(&state.transform).map_err(|source| StreamCheckpointBuildError::TransformState { worker: id, source })?;
366 }
367 if initialized.send((id, Ok(()))).is_err() { return Ok(()); }
368 did_initialize = true;
369 let mut pending = state;
370 loop {
371 let command = select! {
372 recv(commands) -> command => match command { Ok(command) => command, Err(_) => return Ok(()) },
373 recv(shutdown.signal()) -> _ => return Ok(()),
374 };
375 match command {
376 Command::Apply => {
377 if let Some(state) = pending.take() {
378 source.restore_validated(&state.source);
379 transform.restore_validated(&state.transform);
380 }
381 }
382 Command::Begin(run) => {
383 generation = run.generation;
384 source.set_run_context(WorkerContext::new(info, run.cancellation.clone(), run.deadline.clone()));
385 let mut previous = None;
386 loop {
387 let acquired = select! {
388 recv(credits) -> result => result.is_ok(),
389 recv(run.cancellation.signal()) -> _ => false,
390 recv(shutdown.signal()) -> _ => false,
391 };
392 if !acquired || run.cancellation.is_cancelled() || shutdown.is_cancelled() { break; }
393 let boundary = committed.load(Ordering::Acquire);
394 while journal.front().is_some_and(|(attempt, _): &(Attempt, LaneState<S, F>)| attempt.sequence.is_some_and(|sequence| sequence < boundary)) { journal.pop_front(); }
395 assert!(journal.len() < journal_capacity, "validated stream snapshot window exhausted");
396 journal.push_back((Attempt { sequence: None, _logical_id: None, _outcome: AttemptOutcome::InFlight }, StreamLaneState { id, source: source.snapshot(), transform: transform.snapshot() }));
397 let event = match source.next() {
398 None => Event::End,
399 Some(Err(error)) => {
400 let sequence = source.error_sequence(&error).get();
401 if sequence < boundary || previous.is_some_and(|previous| sequence <= previous) {
402 Event::Protocol { sequence: Some(sequence), reason: "exact stream source error has a past or non-increasing sequence".to_owned() }
403 } else { Event::Source { sequence, error } }
404 },
405 Some(Ok(record)) => {
406 let sequence = record.sequence.map(SequenceId::get);
407 match sequence {
408 None => Event::Protocol { sequence, reason: "exact stream record is missing its sequence ID".to_owned() },
409 Some(sequence) if sequence < boundary || previous.is_some_and(|previous| sequence <= previous) => Event::Protocol { sequence: Some(sequence), reason: "exact stream shard emitted a past or non-increasing sequence".to_owned() },
410 Some(sequence) => {
411 previous = Some(sequence);
412 let logical_id = record.logical_id.get();
413 let task = TaskContext { loader_seed: run.loader_seed, epoch: run.epoch, rank: info.rank, logical_sample: logical_id, stage: 0, cancellation: run.cancellation.clone(), deadline: run.deadline.clone() };
414 match transform.transform(record.sample, &task) {
415 Ok(sample) => Event::Record(WorkerRecord { sequence: record.sequence, logical_id: record.logical_id, sample }),
416 Err(error) => Event::Transform { sequence, logical_id, error },
417 }
418 }
419 }
420 }
421 };
422 let (sequence, logical_id, outcome) = match &event {
423 Event::Record(record) => (record.sequence.map(SequenceId::get), Some(record.logical_id.get()), AttemptOutcome::Record),
424 Event::End => (None, None, AttemptOutcome::End),
425 Event::Source { sequence, .. } => (Some(*sequence), None, AttemptOutcome::SourceError),
426 Event::Transform { sequence, logical_id, .. } => (Some(*sequence), Some(*logical_id), AttemptOutcome::TransformError),
427 Event::Protocol { sequence, .. } => (*sequence, None, AttemptOutcome::Protocol),
428 Event::Panic => unreachable!("panic is handled outside production"),
429 };
430 journal.back_mut().expect("pre-attempt snapshot").0 = Attempt { sequence, _logical_id: logical_id, _outcome: outcome };
431 let terminal = !matches!(event, Event::Record(_));
432 let message = Completion { worker: id, generation, event };
433 let sent = select! {
434 send(results, message) -> result => result.is_ok(),
435 recv(run.cancellation.signal()) -> _ => false,
436 recv(shutdown.signal()) -> _ => false,
437 };
438 if terminal || !sent { break; }
439 }
440 }
441 Command::Rollback { generation: requested, boundary } => {
442 let state = if requested != generation { Err(invalid("stale rollback generation").into()) } else {
443 let state = journal.iter().find(|(attempt, _)| attempt.sequence.is_none_or(|sequence| sequence >= boundary)).map(|(_, state)| state.clone())
444 .unwrap_or_else(|| StreamLaneState { id, source: source.snapshot(), transform: transform.snapshot() });
445 source.validate_snapshot(&state.source).map_err(|source| StreamCheckpointBuildError::Source { worker: id, source })
446 .and_then(|()| transform.validate_snapshot(&state.transform).map_err(|source| StreamCheckpointBuildError::TransformState { worker: id, source }))
447 .map(|()| { source.restore_validated(&state.source); transform.restore_validated(&state.transform); state })
448 };
449 journal.clear();
450 while credits.try_recv().is_ok() {}
451 if acknowledgments.send((id, generation, state)).is_err() { return Ok(()); }
452 }
453 }
454 }
455 })));
456 match outcome {
457 Ok(Err(error)) => { let _ = initialized.send((id, Err(error))); }
458 Err(_) if !did_initialize => { let _ = initialized.send((id, Err(StreamCheckpointBuildError::WorkerPanic { worker: id }))); }
459 Err(_) => {
460 let message = Completion { worker: id, generation, event: Event::Panic };
461 select! { send(results, message) -> _ => {}, recv(shutdown.signal()) -> _ => {} }
462 }
463 Ok(Ok(())) => {}
464 }
465 }).map_err(|e| invalid(format!("cannot start exact stream worker: {e}")))?;
466 pool.handles.push(handle);
467 }
468 drop(init_tx);
469 drop(ack_tx);
470 drop(result_tx);
471 let mut seen = vec![false; config.workers];
472 for _ in 0..config.workers {
473 let (id, result) = pool
474 .initialized
475 .recv()
476 .map_err(|_| invalid("worker closed during validation"))?;
477 if id >= seen.len() || std::mem::replace(&mut seen[id], true) {
478 return Err(invalid("duplicate initialization lane").into());
479 }
480 result?;
481 }
482 Ok(pool)
483 }
484
485 fn begin(&self) -> Result<()> {
486 for sender in &self.commands {
487 sender
488 .send(Command::Begin(self.run.clone()))
489 .map_err(|_| invalid("worker closed before production"))?;
490 }
491 Ok(())
492 }
493
494 fn apply(&self) -> Result<()> {
495 for sender in &self.commands {
496 sender
497 .send(Command::Apply)
498 .map_err(|_| invalid("worker closed before validated apply"))?;
499 }
500 Ok(())
501 }
502
503 fn rollback(
504 &mut self,
505 boundary: u64,
506 prefetch: usize,
507 ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
508 let generation = self.run.generation;
509 let next_generation = generation
510 .checked_add(1)
511 .filter(|g| *g < u64::MAX)
512 .ok_or_else(|| invalid("transport generation overflow"))?;
513 self.run.cancel();
514 for command in &self.commands {
515 command
516 .send(Command::Rollback {
517 generation,
518 boundary,
519 })
520 .map_err(|_| invalid("worker failed before rollback"))?;
521 }
522 let states = (0..self.commands.len()).map(|_| None).collect::<Vec<_>>();
523 self.collect_rollback(states, generation, next_generation, boundary, prefetch)
524 }
525
526 fn collect_rollback(
527 &mut self,
528 mut states: Vec<Option<LaneState<S, F>>>,
529 generation: u64,
530 next_generation: u64,
531 boundary: u64,
532 prefetch: usize,
533 ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
534 while states.iter().any(Option::is_none) {
535 select! {
536 recv(self.acknowledgments) -> reply => {
537 let (id, received, state) = reply.map_err(|_| invalid("missing rollback acknowledgment"))?;
538 if id >= states.len() || received != generation || states[id].is_some() { return Err(invalid("stale or duplicate rollback acknowledgment").into()); }
539 states[id] = Some(state?);
540 },
541 recv(self.results) -> message => {
542 if matches!(message, Ok(Completion { event: Event::Panic, .. }) | Err(_)) { return Err(invalid("worker failed before rollback acknowledgment").into()); }
543 },
544 }
545 }
546 self.finish_rollback(states, next_generation, boundary, prefetch)
547 }
548
549 fn finish_rollback(
550 &mut self,
551 states: Vec<Option<LaneState<S, F>>>,
552 generation: u64,
553 boundary: u64,
554 prefetch: usize,
555 ) -> std::result::Result<Vec<LaneState<S, F>>, BuildError<S, F>> {
556 if self.acknowledgments.try_recv().is_ok() {
557 return Err(invalid("duplicate rollback acknowledgment").into());
558 }
559 while self.results.try_recv().is_ok() {}
560 for credit in &self.credits {
561 for _ in 0..prefetch {
562 credit
563 .try_send(())
564 .map_err(|_| invalid("rollback credits were not drained"))?;
565 }
566 }
567 self.committed.store(boundary, Ordering::Release);
568 self.run = WorkerRunContext::new(generation, self.run.loader_seed, self.run.epoch);
569 Ok(states.into_iter().map(Option::unwrap).collect())
570 }
571}
572
573pub struct ExactStreamDataLoader<S, C, F = IdentityTransformFactory, N = PinDisabled>
575where
576 S: WorkerSourceFactory,
577 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
578 F: TransformFactory<S::Sample>,
579 F::Transform: WorkerCheckpoint,
580 C: Collate<TransformOutput<S, F>> + Checkpointable,
581{
582 factory: Arc<S>,
583 transform_factory: Arc<F>,
584 collator: C,
585 configuration: StreamConfiguration,
586 identity: String,
587 pin: PinMemoryStatus,
588 length: Option<usize>,
589 pending: Option<Pool<S, F>>,
590 next_batch: u64,
591 next_sequence: u64,
592 short_tail: bool,
593 source_generation: u64,
594 next_generation: u64,
595 policies: PhantomData<N>,
596}
597
598impl<S, C, F, N> StreamDataLoaderBuilder<S, C, F, NoWorkerInit, MemoryDisabled, N, CheckpointFresh>
599where
600 S: CheckpointSourceFactory,
601 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
602 SourceState<S>: Send + 'static,
603 F: TransformFactory<S::Sample> + Send + Sync + 'static,
604 F::Transform: WorkerCheckpoint + Send + 'static,
605 TransformState<S, F>: Send + 'static,
606 TransformOutput<S, F>: Send + 'static,
607 TransformFailure<S, F>: Send + 'static,
608 F::Error: Send + 'static,
609 C: Collate<TransformOutput<S, F>> + Checkpointable,
610 N: StreamPinPolicy<S, C, F>,
611{
612 pub fn build(self) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>> {
614 let identity = self.checkpoint.identity.clone();
615 build(self, identity, None)
616 }
617}
618
619impl<S, C, F, N>
620 StreamDataLoaderBuilder<
621 S,
622 C,
623 F,
624 NoWorkerInit,
625 MemoryDisabled,
626 N,
627 CheckpointResume<State<S, F, C>>,
628 >
629where
630 S: CheckpointSourceFactory,
631 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
632 SourceState<S>: Send + 'static,
633 F: TransformFactory<S::Sample> + Send + Sync + 'static,
634 F::Transform: WorkerCheckpoint + Send + 'static,
635 TransformState<S, F>: Send + 'static,
636 TransformOutput<S, F>: Send + 'static,
637 TransformFailure<S, F>: Send + 'static,
638 F::Error: Send + 'static,
639 C: Collate<TransformOutput<S, F>> + Checkpointable,
640 N: StreamPinPolicy<S, C, F>,
641{
642 pub fn build(self) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>> {
644 let StreamDataLoaderBuilder {
645 factory,
646 collator,
647 transform_factory,
648 worker_init,
649 configuration,
650 checkpoint,
651 ..
652 } = self;
653 let builder = StreamDataLoaderBuilder {
654 factory,
655 collator,
656 transform_factory,
657 worker_init,
658 configuration,
659 checkpoint: CheckpointDisabled,
660 states: PhantomData,
661 };
662 build(builder, checkpoint.identity, Some(checkpoint.state))
663 }
664}
665
666fn build<S, C, F, N, K>(
667 builder: StreamDataLoaderBuilder<S, C, F, NoWorkerInit, MemoryDisabled, N, K>,
668 identity: String,
669 state: Option<State<S, F, C>>,
670) -> std::result::Result<ExactStreamDataLoader<S, C, F, N>, BuildError<S, F>>
671where
672 S: CheckpointSourceFactory,
673 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
674 SourceState<S>: Send + 'static,
675 F: TransformFactory<S::Sample> + Send + Sync + 'static,
676 F::Transform: WorkerCheckpoint + Send + 'static,
677 TransformState<S, F>: Send + 'static,
678 TransformOutput<S, F>: Send + 'static,
679 TransformFailure<S, F>: Send + 'static,
680 F::Error: Send + 'static,
681 C: Collate<TransformOutput<S, F>> + Checkpointable,
682 N: StreamPinPolicy<S, C, F>,
683{
684 let config = builder.configuration;
685 validate_capacity::<S, F>(config)?;
686 if identity.is_empty() || S::CHECKPOINT_KIND.is_empty() {
687 return Err(invalid("source identity and factory kind must not be empty").into());
688 }
689 let pin = resolve_stream_pin(config.pin_memory)?;
690 if let Some(state) = &state {
691 let batch_size = u64::try_from(config.batch_size)
692 .map_err(|_| invalid("batch size exceeds sequence range"))?;
693 let full = state
694 .next_batch
695 .checked_mul(batch_size)
696 .ok_or_else(|| invalid("batch cursor overflow"))?;
697 let short_tail = !config.drop_last
698 && state.next_batch > 0
699 && state.next_sequence < full
700 && state.next_sequence > full - batch_size;
701 if state.schema_version != 1
702 || state.source_identity != identity
703 || state.factory_kind != S::CHECKPOINT_KIND
704 || state.epoch != config.epoch
705 || state.source_generation > state.transport_generation
706 || state.transport_generation >= u64::MAX - 1
707 || state.rng_derivation_version != TASK_RNG_DERIVATION_VERSION
708 || state.worker_seed_derivation_version != WORKER_SEED_DERIVATION_VERSION
709 || state.configuration != configuration(config, pin)
710 || state.lanes.len() != config.workers
711 || state
712 .lanes
713 .iter()
714 .enumerate()
715 .any(|(id, lane)| lane.id != id)
716 || state.short_tail != short_tail
717 || (state.next_sequence != full && !short_tail)
718 || state.source_length.is_some_and(|length| {
719 state.next_sequence > length as u64
720 || (short_tail && state.next_sequence != length as u64)
721 })
722 {
723 return Err(invalid("incompatible or corrupt stream checkpoint envelope").into());
724 }
725 }
726 let length = builder.factory.exact_len();
727 if state
728 .as_ref()
729 .is_some_and(|state| state.source_length != length)
730 {
731 return Err(invalid("source length changed").into());
732 }
733 let mut loader = ExactStreamDataLoader {
734 factory: Arc::new(builder.factory),
735 transform_factory: Arc::new(builder.transform_factory),
736 collator: builder.collator,
737 configuration: config,
738 identity,
739 pin,
740 length,
741 pending: None,
742 next_batch: state.as_ref().map_or(0, |s| s.next_batch),
743 next_sequence: state.as_ref().map_or(0, |s| s.next_sequence),
744 short_tail: state.as_ref().is_some_and(|s| s.short_tail),
745 source_generation: state.as_ref().map_or(0, |s| s.source_generation),
746 next_generation: state.as_ref().map_or(1, |s| s.transport_generation + 1),
747 policies: PhantomData,
748 };
749 let generation = state.as_ref().map_or(0, |s| s.transport_generation);
750 let pool = Pool::new(
751 Arc::clone(&loader.factory),
752 Arc::clone(&loader.transform_factory),
753 config,
754 loader.source_generation,
755 WorkerRunContext::new(generation, config.loader_seed, config.epoch),
756 loader.next_sequence,
757 state.as_ref().map(|s| s.lanes.clone()),
758 )?;
759 if let Some(state) = &state {
760 loader.collator.validate_state(&state.coordinator)?;
761 pool.apply()?;
762 loader.collator.load_validated(&state.coordinator);
763 }
764 loader.pending = Some(pool);
765 Ok(loader)
766}
767
768pub struct ExactStreamLoaderIter<'a, S, C, F = IdentityTransformFactory, N = PinDisabled>
770where
771 S: WorkerSourceFactory,
772 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
773 F: TransformFactory<S::Sample>,
774 F::Transform: WorkerCheckpoint,
775 C: Collate<TransformOutput<S, F>> + Checkpointable,
776{
777 loader: &'a mut ExactStreamDataLoader<S, C, F, N>,
778 pool: Option<Pool<S, F>>,
779 completed: Vec<Buffered<S, F>>,
780 ended: Vec<bool>,
781 next_sequence: u64,
782 visible_sequence: u64,
783 short_tail: bool,
784 next_batch: u64,
785 stopped: bool,
786 boundary_valid: bool,
787 pending_error: Option<IterError<S, F, C>>,
788}
789
790impl<S, C, F, N> ExactStreamDataLoader<S, C, F, N>
791where
792 S: CheckpointSourceFactory,
793 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
794 SourceState<S>: Send + 'static,
795 F: TransformFactory<S::Sample> + Send + Sync + 'static,
796 F::Transform: WorkerCheckpoint + Send + 'static,
797 TransformState<S, F>: Send + 'static,
798 TransformOutput<S, F>: Send + 'static,
799 TransformFailure<S, F>: Send + 'static,
800 F::Error: Send + 'static,
801 C: Collate<TransformOutput<S, F>> + Checkpointable,
802 N: StreamPinPolicy<S, C, F>,
803{
804 pub fn len(&self) -> Option<usize> {
806 self.length.map(|n| {
807 if self.configuration.drop_last {
808 n / self.configuration.batch_size
809 } else {
810 n.div_ceil(self.configuration.batch_size)
811 }
812 })
813 }
814 pub fn is_empty(&self) -> Option<bool> {
816 self.len().map(|n| n == 0)
817 }
818 pub fn workers(&self) -> usize {
820 self.configuration.workers
821 }
822 pub fn pin_memory_status(&self) -> PinMemoryStatus {
824 self.pin
825 }
826 pub fn set_epoch(&mut self, epoch: u64) {
828 self.pending = None;
829 self.configuration.epoch = epoch;
830 }
831 pub fn iter(&mut self) -> ExactStreamLoaderIter<'_, S, C, F, N> {
833 let retained = self.pending.is_some();
834 let mut pending_error = None;
835 let pool = if retained {
836 self.pending.take()
837 } else {
838 let generation = self.next_generation;
839 if let Some(next) = generation.checked_add(1).filter(|g| *g < u64::MAX) {
840 self.next_generation = next;
841 self.source_generation = generation;
842 match Pool::new(
843 Arc::clone(&self.factory),
844 Arc::clone(&self.transform_factory),
845 self.configuration,
846 generation,
847 WorkerRunContext::new(
848 generation,
849 self.configuration.loader_seed,
850 self.configuration.epoch,
851 ),
852 0,
853 None,
854 ) {
855 Ok(pool) => Some(pool),
856 Err(error) => {
857 pending_error = Some(map_build_error::<S, F, C>(error));
858 None
859 }
860 }
861 } else {
862 pending_error = Some(LoaderError::Configuration(invalid("generation overflow")));
863 None
864 }
865 };
866 if let Some(pool) = &pool
867 && let Err(error) = pool.begin()
868 {
869 pending_error = Some(LoaderError::Configuration(error));
870 }
871 let next_batch = if retained { self.next_batch } else { 0 };
872 let next_sequence = if retained { self.next_sequence } else { 0 };
873 let short_tail = retained && self.short_tail;
874 let ended = vec![false; self.configuration.workers];
875 let completed =
876 Vec::with_capacity(self.configuration.workers * self.configuration.prefetch_factor);
877 ExactStreamLoaderIter {
878 loader: self,
879 pool,
880 completed,
881 ended,
882 next_batch,
883 next_sequence,
884 visible_sequence: next_sequence,
885 short_tail,
886 stopped: false,
887 boundary_valid: pending_error.is_none(),
888 pending_error,
889 }
890 }
891}
892
893fn map_build_error<S, F, C>(error: BuildError<S, F>) -> IterError<S, F, C>
894where
895 S: WorkerSourceFactory,
896 F: TransformFactory<S::Sample>,
897 C: Collate<TransformOutput<S, F>>,
898{
899 match error {
900 StreamCheckpointBuildError::Configuration(e) => LoaderError::Configuration(e),
901 StreamCheckpointBuildError::Source { worker, source } => LoaderError::StreamPipeline {
902 batch: None,
903 worker,
904 sequence: None,
905 logical_id: None,
906 source: PipelineError::Source(source),
907 },
908 StreamCheckpointBuildError::TransformFactory { worker, source } => {
909 LoaderError::StreamPipeline {
910 batch: None,
911 worker,
912 sequence: None,
913 logical_id: None,
914 source: PipelineError::TransformInit(source),
915 }
916 }
917 StreamCheckpointBuildError::TransformState { source, .. } => {
918 LoaderError::Configuration(source)
919 }
920 StreamCheckpointBuildError::WorkerPanic { worker } => LoaderError::WorkerPanic {
921 worker,
922 batch: None,
923 },
924 }
925}
926
927impl<S, C, F, N> ExactStreamLoaderIter<'_, S, C, F, N>
928where
929 S: CheckpointSourceFactory,
930 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
931 SourceState<S>: Send + 'static,
932 F: TransformFactory<S::Sample> + Send + Sync + 'static,
933 F::Transform: WorkerCheckpoint + Send + 'static,
934 TransformState<S, F>: Send + 'static,
935 TransformOutput<S, F>: Send + 'static,
936 TransformFailure<S, F>: Send + 'static,
937 F::Error: Send + 'static,
938 C: Collate<TransformOutput<S, F>> + Checkpointable,
939 N: StreamPinPolicy<S, C, F>,
940{
941 pub fn checkpoint(&mut self) -> std::result::Result<State<S, F, C>, IterError<S, F, C>> {
947 if !self.boundary_valid {
948 return Err(LoaderError::Checkpoint {
949 reason: "no recoverable visible stream boundary".to_owned(),
950 });
951 }
952 let coordinator = self.loader.collator.save_state();
953 let pool = self.pool.as_mut().ok_or_else(|| LoaderError::Checkpoint {
954 reason: "missing worker pool".to_owned(),
955 })?;
956 let lanes = match pool.rollback(
957 self.visible_sequence,
958 self.loader.configuration.prefetch_factor,
959 ) {
960 Ok(lanes) => lanes,
961 Err(error) => {
962 self.boundary_valid = false;
963 self.stopped = true;
964 self.pool = None;
965 return Err(map_build_error::<S, F, C>(error));
966 }
967 };
968 self.completed.clear();
969 self.ended.fill(false);
970 self.next_sequence = self.visible_sequence;
971 self.stopped = false;
972 let state = StreamLoaderState {
973 schema_version: 1,
974 source_identity: self.loader.identity.clone(),
975 factory_kind: S::CHECKPOINT_KIND.to_owned(),
976 epoch: self.loader.configuration.epoch,
977 source_generation: self.loader.source_generation,
978 transport_generation: pool.run.generation,
979 next_batch: self.next_batch,
980 next_sequence: self.visible_sequence,
981 short_tail: self.short_tail,
982 source_length: self.loader.length,
983 rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
984 worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
985 configuration: configuration(self.loader.configuration, self.loader.pin),
986 lanes,
987 coordinator,
988 };
989 self.loader.next_generation = self.loader.next_generation.max(pool.run.generation + 1);
990 pool.begin().map_err(LoaderError::Configuration)?;
991 Ok(state)
992 }
993
994 fn fail(
995 &mut self,
996 error: IterError<S, F, C>,
997 recoverable: bool,
998 ) -> Option<std::result::Result<C::Batch, IterError<S, F, C>>> {
999 self.stopped = true;
1000 self.boundary_valid &= recoverable;
1001 if let Some(pool) = &self.pool {
1002 pool.run.cancel();
1003 }
1004 if !recoverable {
1005 self.pool = None;
1006 }
1007 Some(Err(error))
1008 }
1009}
1010
1011impl<S, C, F, N> Iterator for ExactStreamLoaderIter<'_, S, C, F, N>
1012where
1013 S: CheckpointSourceFactory,
1014 S::Source: CheckpointableSource<Sample = S::Sample, Error = S::Error>,
1015 SourceState<S>: Send + 'static,
1016 F: TransformFactory<S::Sample> + Send + Sync + 'static,
1017 F::Transform: WorkerCheckpoint + Send + 'static,
1018 TransformState<S, F>: Send + 'static,
1019 TransformOutput<S, F>: Send + 'static,
1020 TransformFailure<S, F>: Send + 'static,
1021 F::Error: Send + 'static,
1022 C: Collate<TransformOutput<S, F>> + Checkpointable,
1023 N: StreamPinPolicy<S, C, F>,
1024{
1025 type Item = std::result::Result<C::Batch, IterError<S, F, C>>;
1026 fn next(&mut self) -> Option<Self::Item> {
1027 if self.stopped {
1028 return None;
1029 }
1030 if let Some(error) = self.pending_error.take() {
1031 return self.fail(error, false);
1032 }
1033 let mut samples = Vec::with_capacity(self.loader.configuration.batch_size);
1034 loop {
1035 if let Some(index) = self
1036 .completed
1037 .iter()
1038 .position(|(_, event)| event.sequence() == Some(self.next_sequence))
1039 {
1040 let (worker, event) = self.completed.swap_remove(index);
1041 let record = match event {
1042 Event::Record(record) => record,
1043 Event::Source { sequence, error } => {
1044 return self.fail(
1045 LoaderError::StreamPipeline {
1046 batch: Some(self.next_batch),
1047 worker,
1048 sequence: Some(sequence),
1049 logical_id: None,
1050 source: PipelineError::Source(error),
1051 },
1052 true,
1053 );
1054 }
1055 Event::Transform {
1056 sequence,
1057 logical_id,
1058 error,
1059 } => {
1060 return self.fail(
1061 LoaderError::StreamPipeline {
1062 batch: Some(self.next_batch),
1063 worker,
1064 sequence: Some(sequence),
1065 logical_id: Some(logical_id),
1066 source: PipelineError::Transform(error),
1067 },
1068 true,
1069 );
1070 }
1071 _ => unreachable!("only sequenced outcomes enter reassembly"),
1072 };
1073 let Some(next) = self.next_sequence.checked_add(1) else {
1074 return self.fail(
1075 LoaderError::StreamProtocol {
1076 sequence: Some(self.next_sequence),
1077 reason: "sequence overflow".to_owned(),
1078 },
1079 false,
1080 );
1081 };
1082 self.next_sequence = next;
1083 samples.push(record.sample);
1084 if self.pool.as_ref().expect("active pool").credits[worker]
1085 .try_send(())
1086 .is_err()
1087 {
1088 return self.fail(
1089 LoaderError::ChannelClosed {
1090 batch: self.next_batch,
1091 },
1092 false,
1093 );
1094 }
1095 if samples.len() == self.loader.configuration.batch_size {
1096 break;
1097 }
1098 continue;
1099 }
1100 if self.ended.iter().all(|ended| *ended) {
1101 if !self.completed.is_empty()
1102 || self
1103 .loader
1104 .length
1105 .is_some_and(|length| self.next_sequence != length as u64)
1106 {
1107 return self.fail(
1108 LoaderError::StreamProtocol {
1109 sequence: Some(self.next_sequence),
1110 reason:
1111 "stream ended with a missing global sequence or wrong exact length"
1112 .to_owned(),
1113 },
1114 false,
1115 );
1116 }
1117 if samples.is_empty() || self.loader.configuration.drop_last {
1118 self.stopped = true;
1119 return None;
1120 }
1121 break;
1122 }
1123 if self.ended.iter().enumerate().all(|(id, ended)| {
1125 *ended
1126 || self
1127 .completed
1128 .iter()
1129 .filter(|(worker, _)| *worker == id)
1130 .count()
1131 == self.loader.configuration.prefetch_factor
1132 }) {
1133 return self.fail(
1134 LoaderError::StreamProtocol {
1135 sequence: Some(self.next_sequence),
1136 reason: "bounded stream window contains no expected sequence".to_owned(),
1137 },
1138 false,
1139 );
1140 }
1141 let pool = self.pool.as_ref().expect("active pool");
1142 let completion = match pool.results.recv() {
1143 Ok(completion) => completion,
1144 Err(_) => {
1145 return self.fail(
1146 LoaderError::ChannelClosed {
1147 batch: self.next_batch,
1148 },
1149 false,
1150 );
1151 }
1152 };
1153 if completion.generation != pool.run.generation {
1154 return self.fail(
1155 LoaderError::StreamProtocol {
1156 sequence: None,
1157 reason: "stale transport message".to_owned(),
1158 },
1159 false,
1160 );
1161 }
1162 let worker = completion.worker;
1163 match completion.event {
1164 event @ (Event::Record(_) | Event::Source { .. } | Event::Transform { .. }) => {
1165 let sequence = event.sequence();
1166 if sequence.is_none_or(|sequence| sequence < self.next_sequence)
1167 || self
1168 .completed
1169 .iter()
1170 .any(|(_, previous)| previous.sequence() == sequence)
1171 {
1172 return self.fail(
1173 LoaderError::StreamProtocol {
1174 sequence,
1175 reason: "missing, duplicate, or past sequence ID".to_owned(),
1176 },
1177 false,
1178 );
1179 }
1180 if !matches!(event, Event::Record(_)) {
1181 self.ended[worker] = true;
1182 }
1183 self.completed.push((worker, event));
1184 }
1185 Event::End => {
1186 self.ended[worker] = true;
1187 }
1188 Event::Protocol { sequence, reason } => {
1189 return self.fail(LoaderError::StreamProtocol { sequence, reason }, false);
1190 }
1191 Event::Panic => {
1192 return self.fail(
1193 LoaderError::StreamWorkerPanic {
1194 batch: Some(self.next_batch),
1195 worker,
1196 sequence: None,
1197 logical_id: None,
1198 },
1199 false,
1200 );
1201 }
1202 }
1203 }
1204 let short_tail = samples.len() < self.loader.configuration.batch_size;
1205 let batch = match catch_unwind(AssertUnwindSafe(|| self.loader.collator.collate(samples))) {
1206 Ok(Ok(batch)) => batch,
1207 Ok(Err(source)) => {
1208 return self.fail(
1209 LoaderError::Pipeline {
1210 batch: Some(self.next_batch),
1211 worker: None,
1212 source: PipelineError::Collate(source),
1213 },
1214 false,
1215 );
1216 }
1217 Err(_) => {
1218 return self.fail(
1219 LoaderError::CoordinatorPanic {
1220 stage: "collation",
1221 batch: Some(self.next_batch),
1222 },
1223 false,
1224 );
1225 }
1226 };
1227 let batch = if let PinMemoryStatus::Enabled(device) = self.loader.pin {
1228 match catch_unwind(AssertUnwindSafe(|| N::pin(batch, device))) {
1229 Ok(Ok(batch)) => batch,
1230 Ok(Err(source)) => {
1231 return self.fail(
1232 LoaderError::PinMemory {
1233 batch: self.next_batch,
1234 source,
1235 },
1236 false,
1237 );
1238 }
1239 Err(_) => {
1240 return self.fail(
1241 LoaderError::CoordinatorPanic {
1242 stage: "pinning",
1243 batch: Some(self.next_batch),
1244 },
1245 false,
1246 );
1247 }
1248 }
1249 } else {
1250 batch
1251 };
1252 let Some(next_batch) = self.next_batch.checked_add(1) else {
1253 return self.fail(
1254 LoaderError::Configuration(invalid("batch cursor overflow")),
1255 false,
1256 );
1257 };
1258 self.next_batch = next_batch;
1259 self.visible_sequence = self.next_sequence;
1260 self.short_tail = short_tail;
1261 self.pool
1262 .as_ref()
1263 .expect("active pool")
1264 .committed
1265 .store(self.visible_sequence, Ordering::Release);
1266 Some(Ok(batch))
1267 }
1268}