use incin_core::backend_authoring::Backend;
use incin_core::backend_authoring::{AutogradBackend, HostInterop, VariableBackend};
use incin_core::exec::{LossScaleState, LossScaling};
use incin_core::optim::{Optimizer, ScaledOptimizer};
use incin_core::tensor::base::Tensor;
use incin_core::tensor::device::{DeviceId, DeviceKind, DevicePreference, DeviceSet};
const FASTEST_ORDER: &[DeviceKind] = &[DeviceKind::Cuda, DeviceKind::Wgpu, DeviceKind::Cpu];
pub trait Machine {
fn compiled_in(&self, kind: DeviceKind) -> bool;
fn has_device(&self, device: DeviceId) -> bool;
}
#[derive(Debug, Clone, Copy, Default)]
pub struct HostMachine;
impl Machine for HostMachine {
fn compiled_in(&self, kind: DeviceKind) -> bool {
incin_backends::detect::is_compiled_in(kind)
}
fn has_device(&self, device: DeviceId) -> bool {
if device.ordinal() != 0 {
return false;
}
incin_backends::detect::probe(device.kind()) == Some(device)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Decision {
pub code: &'static str,
pub detail: String,
}
impl Decision {
fn new(code: &'static str, detail: impl Into<String>) -> Self {
Self {
code,
detail: detail.into(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Plan {
devices: DeviceSet,
epochs: usize,
loss_scaling: LossScaling,
decisions: Vec<Decision>,
}
impl Plan {
#[must_use]
pub fn devices(&self) -> &DeviceSet {
&self.devices
}
#[must_use]
pub fn epochs(&self) -> usize {
self.epochs
}
#[must_use]
pub fn loss_scaling(&self) -> LossScaling {
self.loss_scaling
}
#[must_use]
pub fn decisions(&self) -> &[Decision] {
&self.decisions
}
#[must_use]
pub fn is_multi_device(&self) -> bool {
self.devices.is_multi_device()
}
#[must_use]
pub fn explain(&self) -> String {
let mut out = String::new();
out.push_str("Execution Plan:\n");
out.push_str(&format!(
" • Devices: {} device(s) ({})\n",
self.devices.len(),
self.devices.primary().kind().name()
));
out.push_str(&format!(" • Epochs: {}\n", self.epochs));
out.push_str(" • Decisions:\n");
for decision in &self.decisions {
out.push_str(&format!(
" - [{}]: {}\n",
decision.code, decision.detail
));
}
out
}
#[must_use]
pub fn explain_json(&self) -> String {
let decisions: Vec<serde_json::Value> = self
.decisions
.iter()
.map(|d| serde_json::json!({ "code": d.code, "detail": d.detail }))
.collect();
let json = serde_json::json!({
"devices": {
"count": self.devices.len(),
"primary": self.devices.primary().kind().name(),
"is_multi_device": self.is_multi_device(),
},
"epochs": self.epochs,
"decisions": decisions,
});
serde_json::to_string_pretty(&json).unwrap_or_default()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum TrainError {
NotCompiledIn {
kind: DeviceKind,
feature: &'static str,
},
DeviceUnavailable {
device: DeviceId,
},
NoDeviceAvailable,
CollectivesUnavailable {
devices: usize,
},
Step {
epoch: usize,
batch: usize,
message: String,
},
}
impl core::fmt::Display for TrainError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::NotCompiledIn { kind, feature } => write!(
f,
"this build has no {} backend; enable the `{feature}` feature",
kind.name()
),
Self::DeviceUnavailable { device } => write!(
f,
"{}:{} is not available on this machine",
device.kind().name(),
device.ordinal()
),
Self::NoDeviceAvailable => {
f.write_str("no backend is compiled into this build, so there is nowhere to run")
}
Self::CollectivesUnavailable { devices } => write!(
f,
"a {devices}-device run needs collectives, which are not implemented yet (DST-005)"
),
Self::Step {
epoch,
batch,
message,
} => write!(f, "epoch {epoch}, batch {batch}: {message}"),
}
}
}
#[cfg(feature = "std")]
impl std::error::Error for TrainError {}
fn at<T>(
epoch: usize,
batch: usize,
result: incin_core::error::Result<T>,
) -> Result<T, TrainError> {
result.map_err(|error| TrainError::Step {
epoch,
batch,
message: error.to_string(),
})
}
fn device_at(kind: DeviceKind, ordinal: usize) -> Option<DeviceId> {
match kind {
DeviceKind::Cpu => (ordinal == 0).then(DeviceId::cpu),
DeviceKind::Cuda => Some(DeviceId::cuda(ordinal)),
DeviceKind::Wgpu => Some(DeviceId::wgpu(ordinal)),
_ => None,
}
}
const fn feature_for(kind: DeviceKind) -> &'static str {
match kind {
DeviceKind::Cpu => "cpu",
DeviceKind::Cuda => "cuda",
DeviceKind::Wgpu => "wgpu",
_ => "incin-backends",
}
}
#[derive(Debug, Clone)]
pub struct TrainerBuilder {
preference: DevicePreference,
epochs: usize,
loss_scaling: LossScaling,
}
impl Default for TrainerBuilder {
fn default() -> Self {
Self {
preference: DevicePreference::default(),
epochs: 1,
loss_scaling: LossScaling::None,
}
}
}
impl TrainerBuilder {
#[must_use]
pub fn devices(mut self, devices: DeviceSet) -> Self {
self.preference = DevicePreference::Exactly(devices);
self
}
#[must_use]
pub fn device_preference(mut self, preference: DevicePreference) -> Self {
self.preference = preference;
self
}
#[must_use]
pub fn epochs(mut self, epochs: usize) -> Self {
self.epochs = epochs;
self
}
#[must_use]
pub fn loss_scaling(mut self, loss_scaling: LossScaling) -> Self {
self.loss_scaling = loss_scaling;
self
}
pub fn build(self) -> Result<Plan, TrainError> {
self.build_on(&HostMachine)
}
pub fn build_on<M: Machine + ?Sized>(self, machine: &M) -> Result<Plan, TrainError> {
let mut decisions = Vec::new();
let devices = match &self.preference {
DevicePreference::Exactly(requested) => {
let kind = requested.primary().kind();
if !machine.compiled_in(kind) {
return Err(TrainError::NotCompiledIn {
kind,
feature: feature_for(kind),
});
}
for &device in requested.devices() {
if !machine.has_device(device) {
return Err(TrainError::DeviceUnavailable { device });
}
}
decisions.push(Decision::new(
"devices-requested",
format!(
"{} {} device(s), named by the caller",
requested.len(),
kind.name()
),
));
requested.clone()
}
DevicePreference::Cpu => {
let device = DeviceId::cpu();
if !machine.compiled_in(DeviceKind::Cpu) {
return Err(TrainError::NotCompiledIn {
kind: DeviceKind::Cpu,
feature: "cpu",
});
}
if !machine.has_device(device) {
return Err(TrainError::DeviceUnavailable { device });
}
decisions.push(Decision::new("devices-requested", "the CPU, by request"));
DeviceSet::cpu()
}
DevicePreference::Fastest => {
let mut chosen = None;
for &kind in FASTEST_ORDER {
if !machine.compiled_in(kind) {
decisions.push(Decision::new(
"family-not-compiled",
format!("{} skipped: not compiled into this build", kind.name()),
));
continue;
}
let Some(device) = device_at(kind, 0) else {
continue;
};
if !machine.has_device(device) {
decisions.push(Decision::new(
"family-unavailable",
format!("{} skipped: compiled in but not present", kind.name()),
));
continue;
}
chosen = Some(device);
break;
}
let device = chosen.ok_or(TrainError::NoDeviceAvailable)?;
decisions.push(Decision::new(
"devices-resolved",
format!(
"{}:{} chosen as the fastest available family",
device.kind().name(),
device.ordinal()
),
));
DeviceSet::new([device]).unwrap_or_else(|_| DeviceSet::cpu())
}
_ => return Err(TrainError::NoDeviceAvailable),
};
if devices.is_multi_device() {
decisions.push(Decision::new(
"collectives-required",
format!(
"{} devices need collectives; DST-005 has not built them, so this plan \
describes a run it cannot execute",
devices.len()
),
));
}
decisions.push(Decision::new(
"epochs",
format!("{} pass(es) over the data", self.epochs),
));
decisions.push(Decision::new(
"loss-scaling",
format!("{:?} policy configured", self.loss_scaling),
));
Ok(Plan {
devices,
epochs: self.epochs,
loss_scaling: self.loss_scaling,
decisions,
})
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct FitOutcome {
pub epochs: usize,
pub batches: usize,
pub final_loss: Option<f32>,
}
#[derive(Debug, Clone)]
pub struct Trainer {
plan: Plan,
}
impl Trainer {
#[must_use]
pub fn plan() -> TrainerBuilder {
TrainerBuilder::default()
}
#[must_use]
pub fn new(plan: Plan) -> Self {
Self { plan }
}
#[must_use]
pub fn report(&self) -> &Plan {
&self.plan
}
pub fn fit<B, M, O, D, Batch, F>(
&self,
model: &mut M,
optimizer: &mut O,
data: D,
mut loss: F,
) -> Result<FitOutcome, TrainError>
where
B: Backend + VariableBackend + AutogradBackend + HostInterop,
O: Optimizer<B>,
D: IntoIterator<Item = Batch> + Clone,
F: FnMut(
&mut M,
Batch,
) -> incin_core::error::Result<
Tensor<incin_core::shapes::Nil, B, f32, incin_core::tensor::grad::Grad>,
>,
{
if self.plan.is_multi_device() {
return Err(TrainError::CollectivesUnavailable {
devices: self.plan.devices.len(),
});
}
let mut batches = 0;
let mut final_loss = None;
for epoch in 0..self.plan.epochs {
for (batch, item) in data.clone().into_iter().enumerate() {
let value = at(epoch, batch, loss(model, item))?;
let grads = at(epoch, batch, value.backward())?;
at(epoch, batch, optimizer.step(&grads))?;
final_loss = Some(at(epoch, batch, value.to_scalar::<f32>())?);
batches += 1;
}
}
Ok(FitOutcome {
epochs: self.plan.epochs,
batches,
final_loss,
})
}
pub fn fit_scaled<B, M, O, D, Batch, F>(
&self,
model: &mut M,
optimizer: &mut O,
scaler: &mut LossScaleState,
data: D,
mut loss: F,
) -> Result<FitOutcome, TrainError>
where
B: Backend
+ VariableBackend
+ AutogradBackend
+ HostInterop
+ incin_core::backend_authoring::Execute<incin_core::exec::catalog::op::MulScalar>
+ incin_core::optim::OptimizerBackend<f32>,
<B as incin_core::backend_authoring::Execute<incin_core::exec::catalog::op::MulScalar>>::Output:
Into<<B as incin_core::backend_authoring::StorageBackend>::Storage<f32>>,
O: ScaledOptimizer<B>,
D: IntoIterator<Item = Batch> + Clone,
F: FnMut(
&mut M,
Batch,
) -> incin_core::error::Result<
Tensor<incin_core::shapes::Nil, B, f32, incin_core::tensor::grad::Grad>,
>,
{
if self.plan.is_multi_device() {
return Err(TrainError::CollectivesUnavailable {
devices: self.plan.devices.len(),
});
}
let mut batches = 0;
let mut final_loss = None;
for epoch in 0..self.plan.epochs {
for (batch, item) in data.clone().into_iter().enumerate() {
let unscaled_loss_tensor = at(epoch, batch, loss(model, item))?;
let current_scale = scaler.scale();
let loss_for_backward = if (current_scale - 1.0).abs() > f32::EPSILON {
at(
epoch,
batch,
unscaled_loss_tensor.mul_scalar(current_scale as f64),
)?
} else {
unscaled_loss_tensor.clone()
};
let mut grads = at(epoch, batch, loss_for_backward.backward())?;
let _stepped = at(epoch, batch, optimizer.step_scaled(&mut grads, scaler))?;
final_loss = Some(at(epoch, batch, unscaled_loss_tensor.to_scalar::<f32>())?);
batches += 1;
}
}
Ok(FitOutcome {
epochs: self.plan.epochs,
batches,
final_loss,
})
}
}