Skip to main content

rusttorch_data/
loader_checkpoint.rs

1use super::*;
2use crate::worker_checkpoint::{Barrier, error, extra_capacity};
3
4// Only the explicit immutable wrappers have a worker replay implementation.
5trait WorkerReplayDataset: DatasetCheckpoint<State = ()> {}
6impl<D: crate::ReplaySafeDataset> WorkerReplayDataset for crate::ReplaySafeMap<D> {}
7impl WorkerReplayDataset for crate::ReplaySafeTensorDataset {}
8
9type TransformState<D, F> =
10    <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State;
11type Active<D, P, C, F> = WorkerCheckpointActive<
12    <P as LoaderPlan<TransformOutput<D, F>, C>>::Iter,
13    <F as TransformFactory<<D as Dataset>::Sample>>::Transform,
14    TransformState<D, F>,
15>;
16type ExactLoader<D, P, C, F, N> = OwnedDataLoader<
17    D,
18    P,
19    C,
20    F,
21    NoWorkerInit,
22    WorkerExecution,
23    MemoryDisabled,
24    N,
25    Active<D, P, C, F>,
26>;
27type IterCheckpoint<D, P, C, F> =
28    WorkerCheckpointIteration<TransformState<D, F>, IterError<D, P, C, F, NoWorkerInit>>;
29type ExactState<D, P, C, F> = LoaderState<
30    (),
31    <P as CheckpointPlan<TransformOutput<D, F>, C>>::State,
32    WorkerTransformState<TransformState<D, F>>,
33    <P as CheckpointPlan<TransformOutput<D, F>, C>>::CoordinatorState,
34>;
35
36fn preflight<D, P, C, F>(
37    configuration: &mut BuilderConfiguration,
38    explicit: ExplicitArguments,
39    identity: &str,
40) -> Result<()>
41where
42    D: Dataset,
43    F: TransformFactory<D::Sample>,
44    F::Transform: WorkerCheckpoint,
45    P: LoaderPlan<TransformOutput<D, F>, C>,
46{
47    validate_configuration(configuration, explicit)?;
48    if identity.is_empty() {
49        return Err(error("dataset identity must not be empty"));
50    }
51    if !configuration.ordered || configuration.persistent_workers || configuration.timeout.is_some()
52    {
53        return Err(error(
54            "exact worker replay requires ordered, nonpersistent workers without timeout",
55        ));
56    }
57    if configuration.workers > 0 {
58        let prefetch = configuration.prefetch_factor.get_or_insert(2);
59        let outstanding = configuration
60            .workers
61            .checked_mul(*prefetch)
62            .ok_or_else(|| error("outstanding capacity overflow"))?;
63        crate::worker::validate_worker_pool_capacity_extra::<D, F, NoWorkerInit, MemoryDisabled>(
64            configuration.workers,
65            *prefetch,
66            outstanding,
67            extra_capacity::<TransformState<D, F>, IterError<D, P, C, F, NoWorkerInit>>(
68                configuration.workers,
69                *prefetch,
70            )?,
71        )?;
72    }
73    Ok(())
74}
75
76fn finish_builder<D, P, C, F, N, K>(
77    builder: DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, K>,
78    checkpoint: Active<D, P, C, F>,
79    pin_memory_status: PinMemoryStatus,
80) -> ExactLoader<D, P, C, F, N>
81where
82    D: Dataset,
83    F: TransformFactory<D::Sample>,
84    F::Transform: WorkerCheckpoint,
85    P: LoaderPlan<TransformOutput<D, F>, C>,
86{
87    let effective_prefetch = builder
88        .configuration
89        .prefetch_factor
90        .and_then(NonZeroUsize::new);
91    let outstanding_capacity =
92        effective_prefetch.map(|factor| builder.configuration.workers * factor.get());
93    OwnedDataLoader {
94        dataset: Arc::new(builder.dataset),
95        plan: builder.plan,
96        collator: builder.collator,
97        transform_factory: Arc::new(builder.transform_factory),
98        worker_init: Arc::new(builder.worker_init),
99        workers: builder.configuration.workers,
100        persistent_workers: false,
101        timeout: None,
102        ordered: true,
103        pin_memory_status,
104        footprint: None,
105        effective_prefetch_bytes: None,
106        effective_prefetch,
107        outstanding_capacity,
108        loader_seed: builder.configuration.loader_seed,
109        rank: builder.configuration.rank,
110        next_generation: 0,
111        persistent_pool: None,
112        checkpoint,
113        policies: PhantomData,
114    }
115}
116
117#[allow(private_bounds, private_interfaces)]
118impl<D, P, C, F, N>
119    DataLoaderBuilder<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, CheckpointFresh>
120where
121    D: WorkerReplayDataset + Send + Sync + 'static,
122    D::Sample: Send + 'static,
123    D::Error: Send + 'static,
124    F: TransformFactory<D::Sample> + Send + Sync + 'static,
125    F::Transform: WorkerCheckpoint + Send + 'static,
126    TransformState<D, F>: Send + 'static,
127    TransformOutput<D, F>: Send + 'static,
128    TransformFailure<D, F>: Send + 'static,
129    F::Error: Send + 'static,
130    P: LoaderPlanConfiguration + CheckpointPlan<TransformOutput<D, F>, C>,
131    N: MapPinPolicy<D, P, C, F>,
132{
133    /// Builds an exact replay loader with explicit immutable dataset evidence.
134    ///
135    /// Positive workers require ordered delivery, no timeout or persistence,
136    /// disabled byte accounting, and no worker initializer. Snapshot heap
137    /// contents are item-bounded, not measured as total resident bytes.
138    pub fn build(mut self) -> Result<ExactLoader<D, P, C, F, N>> {
139        preflight::<D, P, C, F>(
140            &mut self.configuration,
141            self.explicit,
142            &self.checkpoint.identity,
143        )?;
144        let pin = resolve_pin_request(self.configuration.pin_memory)?;
145        let batch_size = validate_batch_size(self.configuration.batch_size)?;
146        self.plan
147            .apply_batch_options(batch_size, self.configuration.drop_last);
148        self.plan.apply_epoch(self.configuration.epoch);
149        let identity = self.plan.checkpoint_identity();
150        validate_distributed_rank(&identity, self.configuration.rank)?;
151        let checkpoint = WorkerCheckpointActive {
152            identity: self.checkpoint.identity.clone(),
153            configuration: checkpoint_configuration(&identity, &self.configuration, pin)?,
154            batches: None,
155            serial: None,
156            barrier: None,
157            next_batch: 0,
158            next_logical_sample: 0,
159            factory_generation: 0,
160            run_generation: 0,
161        };
162        Ok(finish_builder(self, checkpoint, pin))
163    }
164}
165
166#[allow(private_bounds, private_interfaces)]
167impl<D, P, C, F, N, SS, TS, CS>
168    DataLoaderBuilder<
169        D,
170        P,
171        C,
172        F,
173        NoWorkerInit,
174        WorkerExecution,
175        MemoryDisabled,
176        N,
177        CheckpointResume<LoaderState<(), SS, WorkerTransformState<TS>, CS>>,
178    >
179where
180    D: WorkerReplayDataset + Send + Sync + 'static,
181    D::Sample: Send + 'static,
182    D::Error: Send + 'static,
183    F: TransformFactory<D::Sample> + Send + Sync + 'static,
184    F::Transform: WorkerCheckpoint<State = TS> + Send + 'static,
185    TS: Clone + Send + 'static,
186    TransformOutput<D, F>: Send + 'static,
187    TransformFailure<D, F>: Send + 'static,
188    F::Error: Send + 'static,
189    P: LoaderPlanConfiguration
190        + CheckpointPlan<TransformOutput<D, F>, C, State = SS, CoordinatorState = CS>,
191    N: MapPinPolicy<D, P, C, F>,
192{
193    /// Validates every component before restoring retained worker instances.
194    #[allow(clippy::type_complexity)]
195    pub fn build(
196        mut self,
197    ) -> std::result::Result<ExactLoader<D, P, C, F, N>, CheckpointBuildError<F::Error>> {
198        preflight::<D, P, C, F>(
199            &mut self.configuration,
200            self.explicit,
201            &self.checkpoint.identity,
202        )?;
203        let pin = resolve_pin_request(self.configuration.pin_memory)?;
204        let batch_size = validate_batch_size(self.configuration.batch_size)?;
205        let state = &self.checkpoint.state;
206        validate_static_loader_state_envelope(
207            state,
208            &self.checkpoint.identity,
209            &self.configuration,
210            pin,
211            P::AUTOMATIC_BATCHING,
212            state.transform.run_generation,
213        )?;
214        let transform = &state.transform;
215        require_checkpoint_equal("worker_schema_version", transform.schema_version, 1)?;
216        require_checkpoint_equal("workers", transform.workers, self.configuration.workers)?;
217        require_checkpoint_equal(
218            "prefetch_factor",
219            transform.prefetch_factor,
220            self.configuration.prefetch_factor,
221        )?;
222        if transform.factory_generation > transform.run_generation
223            || transform.run_generation == u64::MAX
224        {
225            return Err(error("invalid or exhausted worker generation").into());
226        }
227        let states = match (&transform.lanes, transform.workers) {
228            (WorkerTransformLanes::Serial(_), 0)
229                if transform.factory_generation == 0 && transform.run_generation == 0 =>
230            {
231                None
232            }
233            (WorkerTransformLanes::Workers(lanes), workers)
234                if workers > 0 && lanes.len() == workers =>
235            {
236                for (id, lane) in lanes.iter().enumerate() {
237                    require_checkpoint_equal("worker_lane", lane.id, id)?;
238                }
239                Some(lanes.iter().map(|lane| lane.state.clone()).collect())
240            }
241            _ => return Err(error("worker lane state does not match execution mode").into()),
242        };
243        let identity = P::checkpoint_identity_from_state(
244            &state.sampler,
245            batch_size,
246            self.configuration.drop_last,
247        )?;
248        validate_distributed_rank(&identity, self.configuration.rank)?;
249        let configuration = checkpoint_configuration(&identity, &self.configuration, pin)?;
250        validate_plan_loader_state_envelope(state, &configuration)?;
251        self.dataset.validate_dataset_state(&state.dataset)?;
252        self.plan.validate_checkpoint_state_with_batch_options(
253            &state.sampler,
254            state.epoch,
255            state.next_batch,
256            state.next_logical_sample,
257            batch_size,
258            self.configuration.drop_last,
259        )?;
260
261        let checkpoint = WorkerCheckpointActive {
262            identity: self.checkpoint.identity.clone(),
263            configuration,
264            batches: None,
265            serial: None,
266            barrier: None,
267            next_batch: state.next_batch,
268            next_logical_sample: state.next_logical_sample,
269            factory_generation: transform.factory_generation,
270            run_generation: transform.run_generation,
271        };
272        // Move the state out without reconstructing or restoring any component.
273        let DataLoaderBuilder {
274            dataset,
275            plan,
276            collator,
277            transform_factory,
278            worker_init,
279            configuration,
280            explicit,
281            checkpoint: CheckpointResume { state, identity },
282            ..
283        } = self;
284        let builder = DataLoaderBuilder {
285            dataset,
286            plan,
287            collator,
288            transform_factory,
289            worker_init,
290            configuration,
291            explicit,
292            checkpoint: CheckpointFresh { identity },
293            states: PhantomData,
294        };
295        let mut loader = finish_builder(builder, checkpoint, pin);
296        if loader.workers == 0 {
297            let mut transform = loader
298                .transform_factory
299                .create(None)
300                .map_err(CheckpointBuildError::TransformFactory)?;
301            let WorkerTransformLanes::Serial(snapshot) = &state.transform.lanes else {
302                unreachable!()
303            };
304            transform.validate_snapshot(snapshot)?;
305            loader
306                .plan
307                .validate_coordinator(&loader.collator, &state.collate)?;
308            transform.restore_validated(snapshot);
309            loader.checkpoint.serial = Some(transform);
310        } else {
311            let (mut pool, barrier) = loader.create_checkpoint_pool(
312                states,
313                state.next_batch,
314                state.transform.factory_generation,
315            )?;
316            pool.validate_initializations(&barrier.initialized)?;
317            loader
318                .plan
319                .validate_coordinator(&loader.collator, &state.collate)?;
320            // Worker restore starts only after all read-only validations succeeded.
321            barrier.apply()?;
322            loader.persistent_pool = Some(pool);
323            loader.checkpoint.barrier = Some(barrier);
324        }
325        loader
326            .plan
327            .apply_batch_options(batch_size, loader.checkpoint.configuration.drop_last);
328        loader.checkpoint.batches = Some(
329            loader
330                .plan
331                .restore_checkpoint_iter_validated(&state.sampler),
332        );
333        loader
334            .plan
335            .restore_coordinator_validated(&mut loader.collator, &state.collate);
336        loader.next_generation = state
337            .transform
338            .run_generation
339            .checked_add(1)
340            .ok_or_else(|| error("generation overflow"))?;
341        Ok(loader)
342    }
343}
344
345#[allow(private_bounds, private_interfaces)]
346impl<D, P, C, F, N> ExactLoader<D, P, C, F, N>
347where
348    D: WorkerReplayDataset + Send + Sync + 'static,
349    D::Sample: Send + 'static,
350    D::Error: Send + 'static,
351    F: TransformFactory<D::Sample> + Send + Sync + 'static,
352    F::Transform: WorkerCheckpoint + Send + 'static,
353    TransformState<D, F>: Send + 'static,
354    TransformOutput<D, F>: Send + 'static,
355    TransformFailure<D, F>: Send + 'static,
356    F::Error: Send + 'static,
357    P: CheckpointPlan<TransformOutput<D, F>, C>,
358    N: MapPinPolicy<D, P, C, F>,
359{
360    #[allow(clippy::type_complexity)]
361    fn create_checkpoint_pool(
362        &self,
363        states: Option<Vec<TransformState<D, F>>>,
364        boundary: u64,
365        factory_generation: u64,
366    ) -> std::result::Result<
367        (
368            WorkerPool<D, F, NoWorkerInit, MemoryDisabled>,
369            Barrier<TransformState<D, F>>,
370        ),
371        CheckpointBuildError<F::Error>,
372    > {
373        let prefetch = self.effective_prefetch.expect("positive workers").get();
374        let (barrier, mut lanes) = Barrier::new(self.workers, prefetch, boundary, states)?;
375        let configuration = WorkerPoolConfiguration {
376            workers: self.workers,
377            prefetch_factor: prefetch,
378            result_capacity: self.outstanding_capacity.expect("positive workers"),
379            seed_generation: factory_generation,
380            loader_seed: self.loader_seed,
381            rank: self.rank,
382        };
383        let pool = WorkerPool::new_with_hooks(
384            Arc::clone(&self.dataset),
385            Arc::clone(&self.transform_factory),
386            Arc::clone(&self.worker_init),
387            configuration,
388            None,
389            true,
390            |id| lanes[id].take().expect("unique lane"),
391        )?;
392        Ok((pool, barrier))
393    }
394
395    /// Starts the retained resume once, then fresh deterministic worker generations.
396    #[allow(clippy::type_complexity)]
397    pub fn iter(
398        &mut self,
399    ) -> WorkerLoaderIter<'_, D, P, C, F, NoWorkerInit, MemoryDisabled, N, IterCheckpoint<D, P, C, F>>
400    {
401        let resumed = self.checkpoint.batches.is_some();
402        let generation = if self.workers == 0 {
403            0
404        } else if resumed {
405            self.checkpoint.run_generation
406        } else {
407            self.next_generation
408        };
409        let factory_generation = if resumed {
410            self.checkpoint.factory_generation
411        } else {
412            generation
413        };
414        let next_batch = if resumed {
415            self.checkpoint.next_batch
416        } else {
417            0
418        };
419        let next_logical_sample = if resumed {
420            self.checkpoint.next_logical_sample
421        } else {
422            0
423        };
424        let mut pending_error = None;
425        let mut pool = self.persistent_pool.take();
426        let mut barrier = self.checkpoint.barrier.take();
427        let mut serial = self.checkpoint.serial.take();
428        let mut serial_error = None;
429        if self.workers > 0 {
430            match generation.checked_add(1) {
431                Some(next) => self.next_generation = self.next_generation.max(next),
432                None => {
433                    pending_error = Some(LoaderError::Configuration(error("generation overflow")))
434                }
435            }
436            if pool.is_none() && pending_error.is_none() {
437                let created = self
438                    .create_checkpoint_pool(None, next_batch, factory_generation)
439                    .and_then(|(mut pool, barrier)| {
440                        pool.validate_initializations(&barrier.initialized)?;
441                        barrier.apply()?;
442                        Ok((pool, barrier))
443                    });
444                match created {
445                    Ok((created_pool, created_barrier)) => {
446                        pool = Some(created_pool);
447                        barrier = Some(created_barrier);
448                    }
449                    Err(CheckpointBuildError::Configuration(error)) => {
450                        pending_error = Some(LoaderError::Configuration(error))
451                    }
452                    Err(CheckpointBuildError::TransformFactory(source)) => {
453                        pending_error = Some(LoaderError::Pipeline {
454                            batch: None,
455                            worker: None,
456                            source: PipelineError::TransformInit(source),
457                        })
458                    }
459                }
460            }
461        } else if serial.is_none() {
462            match self.transform_factory.create(None) {
463                Ok(transform) => serial = Some(transform),
464                Err(error) => serial_error = Some(error),
465            }
466        }
467        let (epoch, batches) = if resumed {
468            (self.plan.epoch(), self.checkpoint.batches.take())
469        } else {
470            match catch_unwind(AssertUnwindSafe(|| (self.plan.epoch(), self.plan.iter()))) {
471                Ok((epoch, batches)) => (epoch, Some(batches)),
472                Err(_) => {
473                    pending_error = Some(LoaderError::CoordinatorPanic {
474                        stage: P::CREATION_PANIC_STAGE,
475                        batch: None,
476                    });
477                    (0, None)
478                }
479            }
480        };
481        let run = WorkerRunContext::new(generation, self.loader_seed, epoch);
482        if let Some(pool) = &mut pool
483            && pool.start_generation(run.clone()).is_err()
484        {
485            pending_error = Some(LoaderError::ChannelClosed { batch: next_batch });
486        }
487        let checkpoint = WorkerCheckpointIteration {
488            identity: self.checkpoint.identity.clone(),
489            configuration: self.checkpoint.configuration.clone(),
490            barrier,
491            logical_ends: BTreeMap::new(),
492            errors: BTreeMap::new(),
493            next_logical_sample,
494            factory_generation,
495            boundary_valid: pending_error.is_none() && serial_error.is_none(),
496        };
497        let mut iterator = WorkerLoaderIter {
498            dataset: Arc::clone(&self.dataset),
499            plan: &mut self.plan,
500            collator: &mut self.collator,
501            batches,
502            serial_transform: serial,
503            serial_transform_error: serial_error,
504            pool: pool.map(IteratorPool::Owned),
505            completed: BTreeMap::new(),
506            pending_error,
507            generation,
508            run_context: Some(run),
509            workers: self.workers,
510            capacity: self.outstanding_capacity.unwrap_or(0),
511            ordered: true,
512            timeout: None,
513            loader_seed: self.loader_seed,
514            epoch,
515            rank: self.rank,
516            next_submission: next_batch,
517            next_visible: next_batch,
518            next_logical_sample,
519            pending_submission: None,
520            outstanding: 0,
521            source_exhausted: false,
522            submission_closed: false,
523            exhausted: false,
524            pin_memory_status: self.pin_memory_status,
525            _checkpoint: checkpoint,
526            policies: PhantomData,
527        };
528        if iterator.workers > 0 && iterator.pending_error.is_none() {
529            iterator.fill_available();
530        }
531        iterator
532    }
533
534    /// Discards any retained resume and selects the epoch for fresh iteration.
535    pub fn set_epoch(&mut self, epoch: u64) {
536        self.persistent_pool = None;
537        self.checkpoint.barrier = None;
538        self.checkpoint.batches = None;
539        self.checkpoint.serial = None;
540        self.plan.set_epoch(epoch);
541    }
542}
543
544#[allow(private_bounds, private_interfaces)]
545impl<D, P, C, F, N>
546    WorkerLoaderIter<'_, D, P, C, F, NoWorkerInit, MemoryDisabled, N, IterCheckpoint<D, P, C, F>>
547where
548    D: WorkerReplayDataset + Send + Sync + 'static,
549    D::Sample: Send + 'static,
550    D::Error: Send + 'static,
551    F: TransformFactory<D::Sample> + Send + Sync + 'static,
552    F::Transform: WorkerCheckpoint + Send + 'static,
553    TransformState<D, F>: Send + 'static,
554    TransformOutput<D, F>: Send + 'static,
555    TransformFailure<D, F>: Send + 'static,
556    F::Error: Send + 'static,
557    P: CheckpointPlan<TransformOutput<D, F>, C>,
558    N: MapPinPolicy<D, P, C, F>,
559{
560    /// Rolls back unpublished worker work and captures the next visible boundary.
561    ///
562    /// The original iterator remains usable, including repeated checkpoints.
563    /// Blocking callbacks still require cooperative cancellation to join.
564    #[allow(clippy::type_complexity)]
565    pub fn checkpoint(
566        &mut self,
567    ) -> std::result::Result<ExactState<D, P, C, F>, IterError<D, P, C, F, NoWorkerInit>> {
568        if !self._checkpoint.boundary_valid || self.exhausted {
569            return Err(LoaderError::Checkpoint {
570                reason: "iterator is not at a successful visible boundary".to_owned(),
571            });
572        }
573        let next_logical_sample = self._checkpoint.next_logical_sample;
574        let sampler = self
575            .plan
576            .checkpoint_state(self.next_visible, next_logical_sample)
577            .map_err(LoaderError::Configuration)?;
578        self.plan
579            .validate_checkpoint_state(&sampler, self.epoch, self.next_visible, next_logical_sample)
580            .map_err(LoaderError::Configuration)?;
581        let lanes = if self.workers == 0 {
582            WorkerTransformLanes::Serial(
583                self.serial_transform
584                    .as_ref()
585                    .expect("validated serial transform")
586                    .snapshot(),
587            )
588        } else {
589            let next_generation = self
590                .generation
591                .checked_add(1)
592                .filter(|generation| *generation < u64::MAX)
593                .ok_or_else(|| LoaderError::Configuration(error("generation overflow")))?;
594            let rollback = (|| -> Result<Vec<TransformState<D, F>>> {
595                let barrier = self
596                    ._checkpoint
597                    .barrier
598                    .as_ref()
599                    .ok_or_else(|| error("missing barrier"))?;
600                barrier.request(self.generation, self.next_visible)?;
601                let pool = self
602                    .pool
603                    .as_mut()
604                    .ok_or_else(|| error("missing pool"))?
605                    .pool_mut();
606                pool.checkpoint_quiesce(self.generation)
607                    .map_err(|()| error("worker failed before rollback acknowledgment"))?;
608                barrier.collect(self.generation)
609            })();
610            let snapshots = match rollback {
611                Ok(states) => states,
612                Err(error) => {
613                    self._checkpoint.boundary_valid = false;
614                    self.stop_poisoned();
615                    return Err(LoaderError::Configuration(error));
616                }
617            };
618            self.completed.clear();
619            self.pending_error = None;
620            self.pending_submission = None;
621            self._checkpoint.errors.clear();
622            self._checkpoint.logical_ends.clear();
623            self.outstanding = 0;
624            self.next_submission = self.next_visible;
625            self.next_logical_sample = next_logical_sample;
626            self.batches = Some(self.plan.restore_checkpoint_iter_validated(&sampler));
627            self.source_exhausted = false;
628            self.submission_closed = false;
629            self.generation = next_generation;
630            let run = WorkerRunContext::new(next_generation, self.loader_seed, self.epoch);
631            if self
632                .pool
633                .as_mut()
634                .expect("live pool")
635                .pool_mut()
636                .start_generation(run.clone())
637                .is_err()
638            {
639                self._checkpoint.boundary_valid = false;
640                self.stop_poisoned();
641                return Err(LoaderError::Configuration(error(
642                    "worker failed to restart after rollback",
643                )));
644            }
645            self.run_context = Some(run);
646            WorkerTransformLanes::Workers(
647                snapshots
648                    .into_iter()
649                    .enumerate()
650                    .map(|(id, state)| WorkerLaneState { id, state })
651                    .collect(),
652            )
653        };
654        let state = LoaderState {
655            schema_version: LOADER_STATE_SCHEMA_VERSION,
656            dataset_identity: self._checkpoint.identity.clone(),
657            epoch: self.epoch,
658            iterator_generation: self.generation,
659            next_batch: self.next_visible,
660            next_logical_sample,
661            dataset: (),
662            sampler,
663            transform: WorkerTransformState {
664                schema_version: 1,
665                workers: self.workers,
666                prefetch_factor: self._checkpoint.configuration.prefetch_factor,
667                factory_generation: self._checkpoint.factory_generation,
668                run_generation: self.generation,
669                lanes,
670            },
671            collate: self.plan.save_coordinator(self.collator),
672            rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
673            worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
674            configuration: self._checkpoint.configuration.clone(),
675        };
676        if self.workers > 0 {
677            self.fill_available();
678        }
679        Ok(state)
680    }
681}