incin 0.1.0

Machine learning framework for setting fire to dimensionality bugs
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
//! The automatic `Trainer` from `PROPOSALS.md` §2 (`UX-001`).
//!
//! §2's UX principles put this at level 1 of three, "select three devices and
//! let Incin produce and explain a safe plan", and the sentence after the list
//! is what makes it a level rather than a convenience wrapper:
//!
//! > "Easy" must not mean silent CPU transfer, hidden padding, relaxed
//! > determinism, or unbounded autotuning. Every automatic decision is
//! > inspectable and reproducible.
//!
//! So the load-bearing property of this module is not the loop. It is that a
//! [`Trainer`] asked for devices it cannot have refuses to build, and that the
//! [`Plan`] it does build says what was decided and why. A trainer that quietly
//! runs on the CPU when the CUDA devices it was handed are absent is worse than
//! no trainer at all, because the failure mode is a training run that finishes.
//!
//! ```rust
//! # use incin::experimental::training::{Machine, Trainer};
//! # use incin::prelude::{DeviceKind, DeviceId, DeviceSet};
//! # struct ThreeGpus;
//! # impl Machine for ThreeGpus {
//! #     fn compiled_in(&self, kind: DeviceKind) -> bool { kind == DeviceKind::Cuda }
//! #     fn has_device(&self, device: DeviceId) -> bool {
//! #         device.kind() == DeviceKind::Cuda && device.ordinal() < 3
//! #     }
//! # }
//! let plan = Trainer::plan()
//!     .devices(DeviceSet::cuda(0..3).unwrap())
//!     .epochs(10)
//!     .build_on(&ThreeGpus)
//!     .unwrap();
//!
//! assert_eq!(plan.devices().len(), 3);
//! assert!(plan.is_multi_device());
//! ```
//!
//! # What this row does not do
//!
//! `ParallelStrategy` and the plan objective are `DST-011`'s, and `.explain()`
//! as a rendered planning report is `UX-005`'s - which depends on both this row
//! and `DST-011`. Multi-device *execution* needs `DST-005`'s collectives. This
//! module therefore plans and validates a multi-device run and refuses to
//! pretend it can execute one: [`Trainer::fit`] on a multi-device plan is an
//! explicit [`TrainError::CollectivesUnavailable`], not a silent single-GPU run.

use incin_core::backend_authoring::Backend;
use incin_core::backend_authoring::{AutogradBackend, HostInterop, VariableBackend};
use incin_core::exec::{LossScaleState, LossScaling};
use incin_core::optim::{Optimizer, ScaledOptimizer};
use incin_core::tensor::base::Tensor;
use incin_core::tensor::device::{DeviceId, DeviceKind, DevicePreference, DeviceSet};

/// The devices a [`DevicePreference::Fastest`] resolution tries, most capable
/// first.
///
/// The same order `incin_backends::detect::PREFERENCE` uses, restated here
/// rather than imported so that this module's behaviour does not change when a
/// backend crate reorders its own detection for an unrelated reason.
const FASTEST_ORDER: &[DeviceKind] = &[DeviceKind::Cuda, DeviceKind::Wgpu, DeviceKind::Cpu];

// ============================================================================
// The machine
// ============================================================================

/// Everything the planner asks about the hardware it is planning for.
///
/// The entire impure surface, two methods, for the same reason `UX-014` put the
/// doctor's hardware questions behind a trait: this row's own deliverable is
/// that "an unchanged model runs on CPU and on three GPUs", and a test that can
/// only describe the runner it happens to be on cannot check the second half of
/// that. A three-GPU machine costs a unit struct here.
pub trait Machine {
    /// Whether this build contains the backend family at all.
    ///
    /// Independent of whether hardware is present, and asked separately because
    /// the two produce different diagnostics: a missing feature is fixed in
    /// `Cargo.toml`, a missing device is not.
    fn compiled_in(&self, kind: DeviceKind) -> bool;

    /// Whether this specific device is present and usable right now.
    fn has_device(&self, device: DeviceId) -> bool;
}

/// [`Machine`] answered by the machine this process is running on.
#[derive(Debug, Clone, Copy, Default)]
pub struct HostMachine;

impl Machine for HostMachine {
    fn compiled_in(&self, kind: DeviceKind) -> bool {
        incin_backends::detect::is_compiled_in(kind)
    }

    fn has_device(&self, device: DeviceId) -> bool {
        // `detect::probe` answers per family, not per ordinal. It reports
        // whether the family has any usable device. That is the right answer at
        // ordinal 0 and no answer at all above it, so anything higher is
        // reported absent rather than guessed present. Being wrong in this
        // direction fails a build that would have worked; being wrong in the
        // other starts a run that cannot.
        if device.ordinal() != 0 {
            return false;
        }
        incin_backends::detect::probe(device.kind()) == Some(device)
    }
}

// ============================================================================
// The plan
// ============================================================================

/// One decision the planner made, and why.
///
/// Carries a stable `code` for the same reason `cargo incin doctor`'s findings
/// do: a support workflow greps the code and a human reads the detail.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Decision {
    /// Stable identifier for the kind of decision.
    pub code: &'static str,
    /// Human-readable specifics.
    pub detail: String,
}

impl Decision {
    fn new(code: &'static str, detail: impl Into<String>) -> Self {
        Self {
            code,
            detail: detail.into(),
        }
    }
}

/// What the builder decided, before anything runs.
///
/// §2: "The returned build report states the selected strategy, mesh, inserted
/// collectives, per-device memory estimate, tuning policy, and fallback
/// decisions." Of those, this row owns the device selection and the fallback
/// decisions; strategy and mesh are `DST-011`'s and are absent rather than
/// stubbed, because a field naming a strategy nothing can plan is worse than a
/// missing one.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Plan {
    devices: DeviceSet,
    epochs: usize,
    loss_scaling: LossScaling,
    decisions: Vec<Decision>,
}

impl Plan {
    /// The devices this run will use.
    #[must_use]
    pub fn devices(&self) -> &DeviceSet {
        &self.devices
    }

    /// How many passes over the data [`Trainer::fit`] will make.
    #[must_use]
    pub fn epochs(&self) -> usize {
        self.epochs
    }

    /// The loss scaling policy configured for this plan.
    #[must_use]
    pub fn loss_scaling(&self) -> LossScaling {
        self.loss_scaling
    }

    /// Every decision the planner made, in the order it made them.
    #[must_use]
    pub fn decisions(&self) -> &[Decision] {
        &self.decisions
    }

    /// Whether this plan needs collectives to execute.
    #[must_use]
    pub fn is_multi_device(&self) -> bool {
        self.devices.is_multi_device()
    }

    /// Renders a human-readable text explanation of the plan and decisions.
    #[must_use]
    pub fn explain(&self) -> String {
        let mut out = String::new();
        out.push_str("Execution Plan:\n");
        out.push_str(&format!(
            "  • Devices: {} device(s) ({})\n",
            self.devices.len(),
            self.devices.primary().kind().name()
        ));
        out.push_str(&format!("  • Epochs: {}\n", self.epochs));
        out.push_str("  • Decisions:\n");
        for decision in &self.decisions {
            out.push_str(&format!(
                "      - [{}]: {}\n",
                decision.code, decision.detail
            ));
        }
        out
    }

    /// Renders a JSON representation of the plan.
    #[must_use]
    pub fn explain_json(&self) -> String {
        let decisions: Vec<serde_json::Value> = self
            .decisions
            .iter()
            .map(|d| serde_json::json!({ "code": d.code, "detail": d.detail }))
            .collect();
        let json = serde_json::json!({
            "devices": {
                "count": self.devices.len(),
                "primary": self.devices.primary().kind().name(),
                "is_multi_device": self.is_multi_device(),
            },
            "epochs": self.epochs,
            "decisions": decisions,
        });
        serde_json::to_string_pretty(&json).unwrap_or_default()
    }
}

// ============================================================================
// Errors
// ============================================================================

/// Why a [`Trainer`] could not be built or could not run.
///
/// Every variant is a refusal. There is deliberately no variant meaning "ran
/// somewhere other than you asked".
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum TrainError {
    /// The requested backend family is not compiled into this build.
    NotCompiledIn {
        /// The family that was asked for.
        kind: DeviceKind,
        /// The Cargo feature that would add it.
        feature: &'static str,
    },
    /// The requested backend family is compiled in, but this device is absent.
    DeviceUnavailable {
        /// The device that was asked for and is not there.
        device: DeviceId,
    },
    /// [`DevicePreference::Fastest`] found nothing at all.
    ///
    /// Reachable only in a build with no backend compiled in, since the CPU is
    /// last in the preference order and is always present when compiled.
    NoDeviceAvailable,
    /// The plan needs collectives, which `DST-005` has not built yet.
    ///
    /// A distinct variant rather than a generic "unsupported" so that the day
    /// `DST-005` lands, the thing to delete is findable.
    CollectivesUnavailable {
        /// How many devices the plan named.
        devices: usize,
    },
    /// A forward pass, backward pass, or optimizer step failed.
    Step {
        /// The epoch the failure happened in, counting from zero.
        epoch: usize,
        /// The batch within that epoch, counting from zero.
        batch: usize,
        /// What the underlying operation reported.
        message: String,
    },
}

impl core::fmt::Display for TrainError {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        match self {
            Self::NotCompiledIn { kind, feature } => write!(
                f,
                "this build has no {} backend; enable the `{feature}` feature",
                kind.name()
            ),
            Self::DeviceUnavailable { device } => write!(
                f,
                "{}:{} is not available on this machine",
                device.kind().name(),
                device.ordinal()
            ),
            Self::NoDeviceAvailable => {
                f.write_str("no backend is compiled into this build, so there is nowhere to run")
            }
            Self::CollectivesUnavailable { devices } => write!(
                f,
                "a {devices}-device run needs collectives, which are not implemented yet (DST-005)"
            ),
            Self::Step {
                epoch,
                batch,
                message,
            } => write!(f, "epoch {epoch}, batch {batch}: {message}"),
        }
    }
}

#[cfg(feature = "std")]
impl std::error::Error for TrainError {}

/// Attaches the position in the run to whatever went wrong.
///
/// A free function rather than a closure in the loop because it is used at four
/// different result types, and a closure would fix itself to the first.
fn at<T>(
    epoch: usize,
    batch: usize,
    result: incin_core::error::Result<T>,
) -> Result<T, TrainError> {
    result.map_err(|error| TrainError::Step {
        epoch,
        batch,
        message: error.to_string(),
    })
}

/// A device of `kind` at `ordinal`, where that names something.
///
/// `DeviceId`'s constructors are per family, and the preference walk holds a
/// `DeviceKind`. `None` for a family this build does not know how to address,
/// which is the honest answer for a `#[non_exhaustive]` enum.
fn device_at(kind: DeviceKind, ordinal: usize) -> Option<DeviceId> {
    match kind {
        DeviceKind::Cpu => (ordinal == 0).then(DeviceId::cpu),
        DeviceKind::Cuda => Some(DeviceId::cuda(ordinal)),
        DeviceKind::Wgpu => Some(DeviceId::wgpu(ordinal)),
        _ => None,
    }
}

/// The Cargo feature that adds a backend family.
const fn feature_for(kind: DeviceKind) -> &'static str {
    match kind {
        DeviceKind::Cpu => "cpu",
        DeviceKind::Cuda => "cuda",
        DeviceKind::Wgpu => "wgpu",
        // `DeviceKind` is `#[non_exhaustive]` outside `incin-core`. A family
        // added later has no feature name here, and naming the crate is more
        // useful than naming a guess.
        _ => "incin-backends",
    }
}

// ============================================================================
// The builder
// ============================================================================

/// Builds a [`Plan`], and with it a [`Trainer`].
///
/// Separate from `Trainer` because §2's example calls `.build()?`. The
/// validation happens once, before training, and its result is a report the
/// caller can read whether or not they go on to train.
#[derive(Debug, Clone)]
pub struct TrainerBuilder {
    preference: DevicePreference,
    epochs: usize,
    loss_scaling: LossScaling,
}

impl Default for TrainerBuilder {
    fn default() -> Self {
        Self {
            preference: DevicePreference::default(),
            epochs: 1,
            loss_scaling: LossScaling::None,
        }
    }
}

impl TrainerBuilder {
    /// Use exactly these devices, or fail.
    #[must_use]
    pub fn devices(mut self, devices: DeviceSet) -> Self {
        self.preference = DevicePreference::Exactly(devices);
        self
    }

    /// Resolve devices from a preference rather than naming them.
    #[must_use]
    pub fn device_preference(mut self, preference: DevicePreference) -> Self {
        self.preference = preference;
        self
    }

    /// How many passes over the data [`Trainer::fit`] should make. Defaults to
    /// one.
    #[must_use]
    pub fn epochs(mut self, epochs: usize) -> Self {
        self.epochs = epochs;
        self
    }

    /// Configures the loss scaling policy for mixed-precision training.
    #[must_use]
    pub fn loss_scaling(mut self, loss_scaling: LossScaling) -> Self {
        self.loss_scaling = loss_scaling;
        self
    }

    /// Validates the request against this machine.
    ///
    /// # Errors
    ///
    /// [`TrainError::NotCompiledIn`] or [`TrainError::DeviceUnavailable`] when
    /// the request cannot be satisfied, and [`TrainError::NoDeviceAvailable`]
    /// when a [`DevicePreference::Fastest`] resolution finds nothing. Never a
    /// substituted device.
    pub fn build(self) -> Result<Plan, TrainError> {
        self.build_on(&HostMachine)
    }

    /// [`build`](Self::build) against a given machine.
    ///
    /// The real entry point; `build` is this with [`HostMachine`]. Written that
    /// way round so the path a test exercises is the path a program runs.
    ///
    /// # Errors
    ///
    /// As [`build`](Self::build).
    pub fn build_on<M: Machine + ?Sized>(self, machine: &M) -> Result<Plan, TrainError> {
        let mut decisions = Vec::new();
        let devices = match &self.preference {
            DevicePreference::Exactly(requested) => {
                let kind = requested.primary().kind();
                if !machine.compiled_in(kind) {
                    return Err(TrainError::NotCompiledIn {
                        kind,
                        feature: feature_for(kind),
                    });
                }
                for &device in requested.devices() {
                    if !machine.has_device(device) {
                        return Err(TrainError::DeviceUnavailable { device });
                    }
                }
                decisions.push(Decision::new(
                    "devices-requested",
                    format!(
                        "{} {} device(s), named by the caller",
                        requested.len(),
                        kind.name()
                    ),
                ));
                requested.clone()
            }
            DevicePreference::Cpu => {
                let device = DeviceId::cpu();
                if !machine.compiled_in(DeviceKind::Cpu) {
                    return Err(TrainError::NotCompiledIn {
                        kind: DeviceKind::Cpu,
                        feature: "cpu",
                    });
                }
                if !machine.has_device(device) {
                    return Err(TrainError::DeviceUnavailable { device });
                }
                decisions.push(Decision::new("devices-requested", "the CPU, by request"));
                DeviceSet::cpu()
            }
            DevicePreference::Fastest => {
                let mut chosen = None;
                for &kind in FASTEST_ORDER {
                    if !machine.compiled_in(kind) {
                        decisions.push(Decision::new(
                            "family-not-compiled",
                            format!("{} skipped: not compiled into this build", kind.name()),
                        ));
                        continue;
                    }
                    let Some(device) = device_at(kind, 0) else {
                        continue;
                    };
                    if !machine.has_device(device) {
                        decisions.push(Decision::new(
                            "family-unavailable",
                            format!("{} skipped: compiled in but not present", kind.name()),
                        ));
                        continue;
                    }
                    chosen = Some(device);
                    break;
                }
                let device = chosen.ok_or(TrainError::NoDeviceAvailable)?;
                decisions.push(Decision::new(
                    "devices-resolved",
                    format!(
                        "{}:{} chosen as the fastest available family",
                        device.kind().name(),
                        device.ordinal()
                    ),
                ));
                // A one-device set from a resolved device cannot be empty, a
                // duplicate, or mixed.
                DeviceSet::new([device]).unwrap_or_else(|_| DeviceSet::cpu())
            }
            // `DevicePreference` is `#[non_exhaustive]` outside `incin-core`. A
            // variant added later has no resolution rule here, and refusing is
            // the only safe answer: the alternative is running somewhere the
            // caller did not ask for, which is the thing this module exists to
            // prevent.
            _ => return Err(TrainError::NoDeviceAvailable),
        };

        if devices.is_multi_device() {
            decisions.push(Decision::new(
                "collectives-required",
                format!(
                    "{} devices need collectives; DST-005 has not built them, so this plan \
                     describes a run it cannot execute",
                    devices.len()
                ),
            ));
        }
        decisions.push(Decision::new(
            "epochs",
            format!("{} pass(es) over the data", self.epochs),
        ));
        decisions.push(Decision::new(
            "loss-scaling",
            format!("{:?} policy configured", self.loss_scaling),
        ));

        Ok(Plan {
            devices,
            epochs: self.epochs,
            loss_scaling: self.loss_scaling,
            decisions,
        })
    }
}

// ============================================================================
// The trainer
// ============================================================================

/// What a completed [`Trainer::fit`] observed.
#[derive(Debug, Clone, PartialEq)]
pub struct FitOutcome {
    /// How many epochs ran.
    pub epochs: usize,
    /// How many batches were stepped in total, across all epochs.
    pub batches: usize,
    /// The loss of the last batch of the last epoch, if there was one.
    ///
    /// `None` for an empty dataset, which is not an error. It is a dataset
    /// with no batches, and reporting zero batches and no loss says so more
    /// honestly than a loss of `0.0`.
    pub final_loss: Option<f32>,
}

/// The automatic trainer from §2.
///
/// Owns the loop for forward, loss, backward, and step, plus the [`Plan`] that says
/// where it will run. The loss itself stays in caller code, passed to
/// [`fit`](Self::fit): what a model's loss is cannot be derived from the model,
/// and a trainer that guessed would be guessing at the one thing training is.
#[derive(Debug, Clone)]
pub struct Trainer {
    plan: Plan,
}

impl Trainer {
    /// Starts a builder.
    ///
    /// §2's example writes `Trainer::new(model, optimizer)`. The model and
    /// optimizer are not builder state here because nothing in the *plan*
    /// depends on them. Planning is about devices, and taking them early
    /// would mean the builder's type parameters propagated into every error
    /// this module returns.
    #[must_use]
    pub fn plan() -> TrainerBuilder {
        TrainerBuilder::default()
    }

    /// Wraps an already-built plan.
    #[must_use]
    pub fn new(plan: Plan) -> Self {
        Self { plan }
    }

    /// The plan this trainer was built with.
    #[must_use]
    pub fn report(&self) -> &Plan {
        &self.plan
    }

    /// Runs the training loop.
    ///
    /// `data` is re-iterated once per epoch, which is why it is `Clone`. A
    /// `&DataLoader` is the intended argument and is `Copy`.
    ///
    /// `loss` receives the model and one batch and returns the scalar to
    /// differentiate. It is a closure rather than a trait method because the
    /// loss is the one part of a training step that is genuinely the caller's.
    ///
    /// # Errors
    ///
    /// [`TrainError::CollectivesUnavailable`] if the plan names more than one
    /// device, and [`TrainError::Step`] carrying the epoch and batch if a
    /// forward pass, backward pass, or optimizer step fails.
    pub fn fit<B, M, O, D, Batch, F>(
        &self,
        model: &mut M,
        optimizer: &mut O,
        data: D,
        mut loss: F,
    ) -> Result<FitOutcome, TrainError>
    where
        B: Backend + VariableBackend + AutogradBackend + HostInterop,
        O: Optimizer<B>,
        D: IntoIterator<Item = Batch> + Clone,
        F: FnMut(
            &mut M,
            Batch,
        ) -> incin_core::error::Result<
            Tensor<incin_core::shapes::Nil, B, f32, incin_core::tensor::grad::Grad>,
        >,
    {
        if self.plan.is_multi_device() {
            return Err(TrainError::CollectivesUnavailable {
                devices: self.plan.devices.len(),
            });
        }

        let mut batches = 0;
        let mut final_loss = None;
        for epoch in 0..self.plan.epochs {
            for (batch, item) in data.clone().into_iter().enumerate() {
                let value = at(epoch, batch, loss(model, item))?;
                let grads = at(epoch, batch, value.backward())?;
                at(epoch, batch, optimizer.step(&grads))?;
                final_loss = Some(at(epoch, batch, value.to_scalar::<f32>())?);
                batches += 1;
            }
        }

        Ok(FitOutcome {
            epochs: self.plan.epochs,
            batches,
            final_loss,
        })
    }

    /// Runs the training loop with mixed-precision loss scaling.
    ///
    /// Scales the computed loss before the backward pass, checks gradients for
    /// non-finite overflow (NaN/Inf), unscales gradients in-place, and steps
    /// the optimizer.
    ///
    /// # Errors
    ///
    /// As [`fit`](Self::fit).
    pub fn fit_scaled<B, M, O, D, Batch, F>(
        &self,
        model: &mut M,
        optimizer: &mut O,
        scaler: &mut LossScaleState,
        data: D,
        mut loss: F,
    ) -> Result<FitOutcome, TrainError>
    where
        B: Backend
            + VariableBackend
            + AutogradBackend
            + HostInterop
            + incin_core::backend_authoring::Execute<incin_core::exec::catalog::op::MulScalar>
            + incin_core::optim::OptimizerBackend<f32>,
        <B as incin_core::backend_authoring::Execute<incin_core::exec::catalog::op::MulScalar>>::Output:
            Into<<B as incin_core::backend_authoring::StorageBackend>::Storage<f32>>,
        O: ScaledOptimizer<B>,
        D: IntoIterator<Item = Batch> + Clone,
        F: FnMut(
            &mut M,
            Batch,
        ) -> incin_core::error::Result<
            Tensor<incin_core::shapes::Nil, B, f32, incin_core::tensor::grad::Grad>,
        >,
    {
        if self.plan.is_multi_device() {
            return Err(TrainError::CollectivesUnavailable {
                devices: self.plan.devices.len(),
            });
        }

        let mut batches = 0;
        let mut final_loss = None;
        for epoch in 0..self.plan.epochs {
            for (batch, item) in data.clone().into_iter().enumerate() {
                let unscaled_loss_tensor = at(epoch, batch, loss(model, item))?;
                let current_scale = scaler.scale();
                let loss_for_backward = if (current_scale - 1.0).abs() > f32::EPSILON {
                    at(
                        epoch,
                        batch,
                        unscaled_loss_tensor.mul_scalar(current_scale as f64),
                    )?
                } else {
                    unscaled_loss_tensor.clone()
                };
                let mut grads = at(epoch, batch, loss_for_backward.backward())?;
                let _stepped = at(epoch, batch, optimizer.step_scaled(&mut grads, scaler))?;
                final_loss = Some(at(epoch, batch, unscaled_loss_tensor.to_scalar::<f32>())?);
                batches += 1;
            }
        }

        Ok(FitOutcome {
            epochs: self.plan.epochs,
            batches,
            final_loss,
        })
    }
}