1use super::*;
2use crate::worker_checkpoint::{Barrier, error, extra_capacity};
3
4trait WorkerReplayDataset: DatasetCheckpoint<State = ()> {}
6impl<D: crate::ReplaySafeDataset> WorkerReplayDataset for crate::ReplaySafeMap<D> {}
7impl WorkerReplayDataset for crate::ReplaySafeTensorDataset {}
8
9type TransformState<D, F> =
10 <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State;
11type Active<D, P, C, F> = WorkerCheckpointActive<
12 <P as LoaderPlan<TransformOutput<D, F>, C>>::Iter,
13 <F as TransformFactory<<D as Dataset>::Sample>>::Transform,
14 TransformState<D, F>,
15>;
16type ExactLoader<D, P, C, F, N> = OwnedDataLoader<
17 D,
18 P,
19 C,
20 F,
21 NoWorkerInit,
22 WorkerExecution,
23 MemoryDisabled,
24 N,
25 Active<D, P, C, F>,
26>;
27type IterCheckpoint<D, P, C, F> =
28 WorkerCheckpointIteration<TransformState<D, F>, IterError<D, P, C, F, NoWorkerInit>>;
29type ExactState<D, P, C, F> = LoaderState<
30 (),
31 <P as CheckpointPlan<TransformOutput<D, F>, C>>::State,
32 WorkerTransformState<TransformState<D, F>>,
33 <P as CheckpointPlan<TransformOutput<D, F>, C>>::CoordinatorState,
34>;
35
36fn preflight<D, P, C, F>(
37 configuration: &mut BuilderConfiguration,
38 explicit: ExplicitArguments,
39 identity: &str,
40) -> Result<()>
41where
42 D: Dataset,
43 F: TransformFactory<D::Sample>,
44 F::Transform: WorkerCheckpoint,
45 P: LoaderPlan<TransformOutput<D, F>, C>,
46{
47 validate_configuration(configuration, explicit)?;
48 if identity.is_empty() {
49 return Err(error("dataset identity must not be empty"));
50 }
51 if !configuration.ordered || configuration.persistent_workers || configuration.timeout.is_some()
52 {
53 return Err(error(
54 "exact worker replay requires ordered, nonpersistent workers without timeout",
55 ));
56 }
57 if configuration.workers > 0 {
58 let prefetch = configuration.prefetch_factor.get_or_insert(2);
59 let outstanding = configuration
60 .workers
61 .checked_mul(*prefetch)
62 .ok_or_else(|| error("outstanding capacity overflow"))?;
63 crate::worker::validate_worker_pool_capacity_extra::<D, F, NoWorkerInit, MemoryDisabled>(
64 configuration.workers,
65 *prefetch,
66 outstanding,
67 extra_capacity::<TransformState<D, F>, IterError<D, P, C, F, NoWorkerInit>>(
68 configuration.workers,
69 *prefetch,
70 )?,
71 )?;
72 }
73 Ok(())
74}
75
76fn finish_builder<D, P, C, F, N, K>(
77 builder: DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, K>,
78 checkpoint: Active<D, P, C, F>,
79 pin_memory_status: PinMemoryStatus,
80) -> ExactLoader<D, P, C, F, N>
81where
82 D: Dataset,
83 F: TransformFactory<D::Sample>,
84 F::Transform: WorkerCheckpoint,
85 P: LoaderPlan<TransformOutput<D, F>, C>,
86{
87 let effective_prefetch = builder
88 .configuration
89 .prefetch_factor
90 .and_then(NonZeroUsize::new);
91 let outstanding_capacity =
92 effective_prefetch.map(|factor| builder.configuration.workers * factor.get());
93 OwnedDataLoader {
94 dataset: Arc::new(builder.dataset),
95 plan: builder.plan,
96 collator: builder.collator,
97 transform_factory: Arc::new(builder.transform_factory),
98 worker_init: Arc::new(builder.worker_init),
99 workers: builder.configuration.workers,
100 persistent_workers: false,
101 timeout: None,
102 ordered: true,
103 pin_memory_status,
104 footprint: None,
105 effective_prefetch_bytes: None,
106 effective_prefetch,
107 outstanding_capacity,
108 loader_seed: builder.configuration.loader_seed,
109 rank: builder.configuration.rank,
110 next_generation: 0,
111 persistent_pool: None,
112 checkpoint,
113 policies: PhantomData,
114 }
115}
116
117#[allow(private_bounds, private_interfaces)]
118impl<D, P, C, F, N>
119 DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, CheckpointFresh>
120where
121 D: WorkerReplayDataset + Send + Sync + 'static,
122 D::Sample: Send + 'static,
123 D::Error: Send + 'static,
124 F: TransformFactory<D::Sample> + Send + Sync + 'static,
125 F::Transform: WorkerCheckpoint + Send + 'static,
126 TransformState<D, F>: Send + 'static,
127 TransformOutput<D, F>: Send + 'static,
128 TransformFailure<D, F>: Send + 'static,
129 F::Error: Send + 'static,
130 P: LoaderPlanConfiguration + CheckpointPlan<TransformOutput<D, F>, C>,
131 N: MapPinPolicy<D, P, C, F>,
132{
133 pub fn build(mut self) -> Result<ExactLoader<D, P, C, F, N>> {
139 preflight::<D, P, C, F>(
140 &mut self.configuration,
141 self.explicit,
142 &self.checkpoint.identity,
143 )?;
144 let pin = resolve_pin_request(self.configuration.pin_memory)?;
145 let batch_size = validate_batch_size(self.configuration.batch_size)?;
146 self.plan
147 .apply_batch_options(batch_size, self.configuration.drop_last);
148 self.plan.apply_epoch(self.configuration.epoch);
149 let identity = self.plan.checkpoint_identity();
150 validate_distributed_rank(&identity, self.configuration.rank)?;
151 let checkpoint = WorkerCheckpointActive {
152 identity: self.checkpoint.identity.clone(),
153 configuration: checkpoint_configuration(&identity, &self.configuration, pin)?,
154 batches: None,
155 serial: None,
156 barrier: None,
157 next_batch: 0,
158 next_logical_sample: 0,
159 factory_generation: 0,
160 run_generation: 0,
161 };
162 Ok(finish_builder(self, checkpoint, pin))
163 }
164}
165
166#[allow(private_bounds, private_interfaces)]
167impl<D, P, C, F, N, SS, TS, CS>
168 DataLoaderBuilder<
169 D,
170 P,
171 C,
172 F,
173 NoWorkerInit,
174 WorkerExecution,
175 MemoryDisabled,
176 N,
177 CheckpointResume<LoaderState<(), SS, WorkerTransformState<TS>, CS>>,
178 >
179where
180 D: WorkerReplayDataset + Send + Sync + 'static,
181 D::Sample: Send + 'static,
182 D::Error: Send + 'static,
183 F: TransformFactory<D::Sample> + Send + Sync + 'static,
184 F::Transform: WorkerCheckpoint<State = TS> + Send + 'static,
185 TS: Clone + Send + 'static,
186 TransformOutput<D, F>: Send + 'static,
187 TransformFailure<D, F>: Send + 'static,
188 F::Error: Send + 'static,
189 P: LoaderPlanConfiguration
190 + CheckpointPlan<TransformOutput<D, F>, C, State = SS, CoordinatorState = CS>,
191 N: MapPinPolicy<D, P, C, F>,
192{
193 #[allow(clippy::type_complexity)]
195 pub fn build(
196 mut self,
197 ) -> std::result::Result<ExactLoader<D, P, C, F, N>, CheckpointBuildError<F::Error>> {
198 preflight::<D, P, C, F>(
199 &mut self.configuration,
200 self.explicit,
201 &self.checkpoint.identity,
202 )?;
203 let pin = resolve_pin_request(self.configuration.pin_memory)?;
204 let batch_size = validate_batch_size(self.configuration.batch_size)?;
205 let state = &self.checkpoint.state;
206 validate_static_loader_state_envelope(
207 state,
208 &self.checkpoint.identity,
209 &self.configuration,
210 pin,
211 P::AUTOMATIC_BATCHING,
212 state.transform.run_generation,
213 )?;
214 let transform = &state.transform;
215 require_checkpoint_equal("worker_schema_version", transform.schema_version, 1)?;
216 require_checkpoint_equal("workers", transform.workers, self.configuration.workers)?;
217 require_checkpoint_equal(
218 "prefetch_factor",
219 transform.prefetch_factor,
220 self.configuration.prefetch_factor,
221 )?;
222 if transform.factory_generation > transform.run_generation
223 || transform.run_generation == u64::MAX
224 {
225 return Err(error("invalid or exhausted worker generation").into());
226 }
227 let states = match (&transform.lanes, transform.workers) {
228 (WorkerTransformLanes::Serial(_), 0)
229 if transform.factory_generation == 0 && transform.run_generation == 0 =>
230 {
231 None
232 }
233 (WorkerTransformLanes::Workers(lanes), workers)
234 if workers > 0 && lanes.len() == workers =>
235 {
236 for (id, lane) in lanes.iter().enumerate() {
237 require_checkpoint_equal("worker_lane", lane.id, id)?;
238 }
239 Some(lanes.iter().map(|lane| lane.state.clone()).collect())
240 }
241 _ => return Err(error("worker lane state does not match execution mode").into()),
242 };
243 let identity = P::checkpoint_identity_from_state(
244 &state.sampler,
245 batch_size,
246 self.configuration.drop_last,
247 )?;
248 validate_distributed_rank(&identity, self.configuration.rank)?;
249 let configuration = checkpoint_configuration(&identity, &self.configuration, pin)?;
250 validate_plan_loader_state_envelope(state, &configuration)?;
251 self.dataset.validate_dataset_state(&state.dataset)?;
252 self.plan.validate_checkpoint_state_with_batch_options(
253 &state.sampler,
254 state.epoch,
255 state.next_batch,
256 state.next_logical_sample,
257 batch_size,
258 self.configuration.drop_last,
259 )?;
260
261 let checkpoint = WorkerCheckpointActive {
262 identity: self.checkpoint.identity.clone(),
263 configuration,
264 batches: None,
265 serial: None,
266 barrier: None,
267 next_batch: state.next_batch,
268 next_logical_sample: state.next_logical_sample,
269 factory_generation: transform.factory_generation,
270 run_generation: transform.run_generation,
271 };
272 let DataLoaderBuilder {
274 dataset,
275 plan,
276 collator,
277 transform_factory,
278 worker_init,
279 configuration,
280 explicit,
281 checkpoint: CheckpointResume { state, identity },
282 ..
283 } = self;
284 let builder = DataLoaderBuilder {
285 dataset,
286 plan,
287 collator,
288 transform_factory,
289 worker_init,
290 configuration,
291 explicit,
292 checkpoint: CheckpointFresh { identity },
293 states: PhantomData,
294 };
295 let mut loader = finish_builder(builder, checkpoint, pin);
296 if loader.workers == 0 {
297 let mut transform = loader
298 .transform_factory
299 .create(None)
300 .map_err(CheckpointBuildError::TransformFactory)?;
301 let WorkerTransformLanes::Serial(snapshot) = &state.transform.lanes else {
302 unreachable!()
303 };
304 transform.validate_snapshot(snapshot)?;
305 loader
306 .plan
307 .validate_coordinator(&loader.collator, &state.collate)?;
308 transform.restore_validated(snapshot);
309 loader.checkpoint.serial = Some(transform);
310 } else {
311 let (mut pool, barrier) = loader.create_checkpoint_pool(
312 states,
313 state.next_batch,
314 state.transform.factory_generation,
315 )?;
316 pool.validate_initializations(&barrier.initialized)?;
317 loader
318 .plan
319 .validate_coordinator(&loader.collator, &state.collate)?;
320 barrier.apply()?;
322 loader.persistent_pool = Some(pool);
323 loader.checkpoint.barrier = Some(barrier);
324 }
325 loader
326 .plan
327 .apply_batch_options(batch_size, loader.checkpoint.configuration.drop_last);
328 loader.checkpoint.batches = Some(
329 loader
330 .plan
331 .restore_checkpoint_iter_validated(&state.sampler),
332 );
333 loader
334 .plan
335 .restore_coordinator_validated(&mut loader.collator, &state.collate);
336 loader.next_generation = state
337 .transform
338 .run_generation
339 .checked_add(1)
340 .ok_or_else(|| error("generation overflow"))?;
341 Ok(loader)
342 }
343}
344
345#[allow(private_bounds, private_interfaces)]
346impl<D, P, C, F, N> ExactLoader<D, P, C, F, N>
347where
348 D: WorkerReplayDataset + Send + Sync + 'static,
349 D::Sample: Send + 'static,
350 D::Error: Send + 'static,
351 F: TransformFactory<D::Sample> + Send + Sync + 'static,
352 F::Transform: WorkerCheckpoint + Send + 'static,
353 TransformState<D, F>: Send + 'static,
354 TransformOutput<D, F>: Send + 'static,
355 TransformFailure<D, F>: Send + 'static,
356 F::Error: Send + 'static,
357 P: CheckpointPlan<TransformOutput<D, F>, C>,
358 N: MapPinPolicy<D, P, C, F>,
359{
360 #[allow(clippy::type_complexity)]
361 fn create_checkpoint_pool(
362 &self,
363 states: Option<Vec<TransformState<D, F>>>,
364 boundary: u64,
365 factory_generation: u64,
366 ) -> std::result::Result<
367 (
368 WorkerPool<D, F, NoWorkerInit, MemoryDisabled>,
369 Barrier<TransformState<D, F>>,
370 ),
371 CheckpointBuildError<F::Error>,
372 > {
373 let prefetch = self.effective_prefetch.expect("positive workers").get();
374 let (barrier, mut lanes) = Barrier::new(self.workers, prefetch, boundary, states)?;
375 let configuration = WorkerPoolConfiguration {
376 workers: self.workers,
377 prefetch_factor: prefetch,
378 result_capacity: self.outstanding_capacity.expect("positive workers"),
379 seed_generation: factory_generation,
380 loader_seed: self.loader_seed,
381 rank: self.rank,
382 };
383 let pool = WorkerPool::new_with_hooks(
384 Arc::clone(&self.dataset),
385 Arc::clone(&self.transform_factory),
386 Arc::clone(&self.worker_init),
387 configuration,
388 None,
389 true,
390 |id| lanes[id].take().expect("unique lane"),
391 )?;
392 Ok((pool, barrier))
393 }
394
395 #[allow(clippy::type_complexity)]
397 pub fn iter(
398 &mut self,
399 ) -> WorkerLoaderIter<'_, D, P, C, F, NoWorkerInit, MemoryDisabled, N, IterCheckpoint<D, P, C, F>>
400 {
401 let resumed = self.checkpoint.batches.is_some();
402 let generation = if self.workers == 0 {
403 0
404 } else if resumed {
405 self.checkpoint.run_generation
406 } else {
407 self.next_generation
408 };
409 let factory_generation = if resumed {
410 self.checkpoint.factory_generation
411 } else {
412 generation
413 };
414 let next_batch = if resumed {
415 self.checkpoint.next_batch
416 } else {
417 0
418 };
419 let next_logical_sample = if resumed {
420 self.checkpoint.next_logical_sample
421 } else {
422 0
423 };
424 let mut pending_error = None;
425 let mut pool = self.persistent_pool.take();
426 let mut barrier = self.checkpoint.barrier.take();
427 let mut serial = self.checkpoint.serial.take();
428 let mut serial_error = None;
429 if self.workers > 0 {
430 match generation.checked_add(1) {
431 Some(next) => self.next_generation = self.next_generation.max(next),
432 None => {
433 pending_error = Some(LoaderError::Configuration(error("generation overflow")))
434 }
435 }
436 if pool.is_none() && pending_error.is_none() {
437 let created = self
438 .create_checkpoint_pool(None, next_batch, factory_generation)
439 .and_then(|(mut pool, barrier)| {
440 pool.validate_initializations(&barrier.initialized)?;
441 barrier.apply()?;
442 Ok((pool, barrier))
443 });
444 match created {
445 Ok((created_pool, created_barrier)) => {
446 pool = Some(created_pool);
447 barrier = Some(created_barrier);
448 }
449 Err(CheckpointBuildError::Configuration(error)) => {
450 pending_error = Some(LoaderError::Configuration(error))
451 }
452 Err(CheckpointBuildError::TransformFactory(source)) => {
453 pending_error = Some(LoaderError::Pipeline {
454 batch: None,
455 worker: None,
456 source: PipelineError::TransformInit(source),
457 })
458 }
459 }
460 }
461 } else if serial.is_none() {
462 match self.transform_factory.create(None) {
463 Ok(transform) => serial = Some(transform),
464 Err(error) => serial_error = Some(error),
465 }
466 }
467 let (epoch, batches) = if resumed {
468 (self.plan.epoch(), self.checkpoint.batches.take())
469 } else {
470 match catch_unwind(AssertUnwindSafe(|| (self.plan.epoch(), self.plan.iter()))) {
471 Ok((epoch, batches)) => (epoch, Some(batches)),
472 Err(_) => {
473 pending_error = Some(LoaderError::CoordinatorPanic {
474 stage: P::CREATION_PANIC_STAGE,
475 batch: None,
476 });
477 (0, None)
478 }
479 }
480 };
481 let run = WorkerRunContext::new(generation, self.loader_seed, epoch);
482 if let Some(pool) = &mut pool
483 && pool.start_generation(run.clone()).is_err()
484 {
485 pending_error = Some(LoaderError::ChannelClosed { batch: next_batch });
486 }
487 let checkpoint = WorkerCheckpointIteration {
488 identity: self.checkpoint.identity.clone(),
489 configuration: self.checkpoint.configuration.clone(),
490 barrier,
491 logical_ends: BTreeMap::new(),
492 errors: BTreeMap::new(),
493 next_logical_sample,
494 factory_generation,
495 boundary_valid: pending_error.is_none() && serial_error.is_none(),
496 };
497 let mut iterator = WorkerLoaderIter {
498 dataset: Arc::clone(&self.dataset),
499 plan: &mut self.plan,
500 collator: &mut self.collator,
501 batches,
502 serial_transform: serial,
503 serial_transform_error: serial_error,
504 pool: pool.map(IteratorPool::Owned),
505 completed: BTreeMap::new(),
506 pending_error,
507 generation,
508 run_context: Some(run),
509 workers: self.workers,
510 capacity: self.outstanding_capacity.unwrap_or(0),
511 ordered: true,
512 timeout: None,
513 loader_seed: self.loader_seed,
514 epoch,
515 rank: self.rank,
516 next_submission: next_batch,
517 next_visible: next_batch,
518 next_logical_sample,
519 pending_submission: None,
520 outstanding: 0,
521 source_exhausted: false,
522 submission_closed: false,
523 exhausted: false,
524 pin_memory_status: self.pin_memory_status,
525 _checkpoint: checkpoint,
526 policies: PhantomData,
527 };
528 if iterator.workers > 0 && iterator.pending_error.is_none() {
529 iterator.fill_available();
530 }
531 iterator
532 }
533
534 pub fn set_epoch(&mut self, epoch: u64) {
536 self.persistent_pool = None;
537 self.checkpoint.barrier = None;
538 self.checkpoint.batches = None;
539 self.checkpoint.serial = None;
540 self.plan.set_epoch(epoch);
541 }
542}
543
544#[allow(private_bounds, private_interfaces)]
545impl<D, P, C, F, N>
546 WorkerLoaderIter<'_, D, P, C, F, NoWorkerInit, MemoryDisabled, N, IterCheckpoint<D, P, C, F>>
547where
548 D: WorkerReplayDataset + Send + Sync + 'static,
549 D::Sample: Send + 'static,
550 D::Error: Send + 'static,
551 F: TransformFactory<D::Sample> + Send + Sync + 'static,
552 F::Transform: WorkerCheckpoint + Send + 'static,
553 TransformState<D, F>: Send + 'static,
554 TransformOutput<D, F>: Send + 'static,
555 TransformFailure<D, F>: Send + 'static,
556 F::Error: Send + 'static,
557 P: CheckpointPlan<TransformOutput<D, F>, C>,
558 N: MapPinPolicy<D, P, C, F>,
559{
560 #[allow(clippy::type_complexity)]
565 pub fn checkpoint(
566 &mut self,
567 ) -> std::result::Result<ExactState<D, P, C, F>, IterError<D, P, C, F, NoWorkerInit>> {
568 if !self._checkpoint.boundary_valid || self.exhausted {
569 return Err(LoaderError::Checkpoint {
570 reason: "iterator is not at a successful visible boundary".to_owned(),
571 });
572 }
573 let next_logical_sample = self._checkpoint.next_logical_sample;
574 let sampler = self
575 .plan
576 .checkpoint_state(self.next_visible, next_logical_sample)
577 .map_err(LoaderError::Configuration)?;
578 self.plan
579 .validate_checkpoint_state(&sampler, self.epoch, self.next_visible, next_logical_sample)
580 .map_err(LoaderError::Configuration)?;
581 let lanes = if self.workers == 0 {
582 WorkerTransformLanes::Serial(
583 self.serial_transform
584 .as_ref()
585 .expect("validated serial transform")
586 .snapshot(),
587 )
588 } else {
589 let next_generation = self
590 .generation
591 .checked_add(1)
592 .filter(|generation| *generation < u64::MAX)
593 .ok_or_else(|| LoaderError::Configuration(error("generation overflow")))?;
594 let rollback = (|| -> Result<Vec<TransformState<D, F>>> {
595 let barrier = self
596 ._checkpoint
597 .barrier
598 .as_ref()
599 .ok_or_else(|| error("missing barrier"))?;
600 barrier.request(self.generation, self.next_visible)?;
601 let pool = self
602 .pool
603 .as_mut()
604 .ok_or_else(|| error("missing pool"))?
605 .pool_mut();
606 pool.checkpoint_quiesce(self.generation)
607 .map_err(|()| error("worker failed before rollback acknowledgment"))?;
608 barrier.collect(self.generation)
609 })();
610 let snapshots = match rollback {
611 Ok(states) => states,
612 Err(error) => {
613 self._checkpoint.boundary_valid = false;
614 self.stop_poisoned();
615 return Err(LoaderError::Configuration(error));
616 }
617 };
618 self.completed.clear();
619 self.pending_error = None;
620 self.pending_submission = None;
621 self._checkpoint.errors.clear();
622 self._checkpoint.logical_ends.clear();
623 self.outstanding = 0;
624 self.next_submission = self.next_visible;
625 self.next_logical_sample = next_logical_sample;
626 self.batches = Some(self.plan.restore_checkpoint_iter_validated(&sampler));
627 self.source_exhausted = false;
628 self.submission_closed = false;
629 self.generation = next_generation;
630 let run = WorkerRunContext::new(next_generation, self.loader_seed, self.epoch);
631 if self
632 .pool
633 .as_mut()
634 .expect("live pool")
635 .pool_mut()
636 .start_generation(run.clone())
637 .is_err()
638 {
639 self._checkpoint.boundary_valid = false;
640 self.stop_poisoned();
641 return Err(LoaderError::Configuration(error(
642 "worker failed to restart after rollback",
643 )));
644 }
645 self.run_context = Some(run);
646 WorkerTransformLanes::Workers(
647 snapshots
648 .into_iter()
649 .enumerate()
650 .map(|(id, state)| WorkerLaneState { id, state })
651 .collect(),
652 )
653 };
654 let state = LoaderState {
655 schema_version: LOADER_STATE_SCHEMA_VERSION,
656 dataset_identity: self._checkpoint.identity.clone(),
657 epoch: self.epoch,
658 iterator_generation: self.generation,
659 next_batch: self.next_visible,
660 next_logical_sample,
661 dataset: (),
662 sampler,
663 transform: WorkerTransformState {
664 schema_version: 1,
665 workers: self.workers,
666 prefetch_factor: self._checkpoint.configuration.prefetch_factor,
667 factory_generation: self._checkpoint.factory_generation,
668 run_generation: self.generation,
669 lanes,
670 },
671 collate: self.plan.save_coordinator(self.collator),
672 rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
673 worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
674 configuration: self._checkpoint.configuration.clone(),
675 };
676 if self.workers > 0 {
677 self.fill_available();
678 }
679 Ok(state)
680 }
681}