use crate::err::{Error, Result};
use crate::exec::{LayoutClass, MathMode};
use crate::shapes::Dyn;
use crate::shapes::error::OperationKind;
use crate::tensor::dtype::{BuiltinDType, ConstDType, DType, DTypeDescriptor, FloatDType};
use crate::tensor::dtype::{bf16, f16};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum PrecisionRole {
Compute,
Accumulator,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct LossScaleState {
policy: LossScaling,
scale: f32,
steps_since_last_overflow: usize,
}
impl LossScaleState {
#[must_use]
pub fn new(policy: LossScaling) -> Self {
let scale = policy.initial_scale();
Self {
policy,
scale,
steps_since_last_overflow: 0,
}
}
#[must_use]
pub const fn policy(&self) -> LossScaling {
self.policy
}
#[must_use]
pub const fn scale(&self) -> f32 {
self.scale
}
#[must_use]
pub const fn steps_since_last_overflow(&self) -> usize {
self.steps_since_last_overflow
}
pub fn update(&mut self, found_nan_or_inf: bool) {
if let LossScaling::Dynamic {
growth_factor,
backoff_factor,
growth_interval,
..
} = self.policy
{
if found_nan_or_inf {
self.scale = (self.scale * backoff_factor).max(1.0);
self.steps_since_last_overflow = 0;
} else {
self.steps_since_last_overflow += 1;
if self.steps_since_last_overflow >= growth_interval {
self.scale *= growth_factor;
self.steps_since_last_overflow = 0;
}
}
}
}
pub fn unscale_and_update_vars<'a, B, K, I>(
&mut self,
vars: I,
grads: &mut crate::autograd::Gradients<B>,
) -> Result<bool>
where
B: crate::tensor::backend::VariableBackend
+ crate::tensor::backend::AutogradBackend
+ crate::optim::OptimizerBackend<K>
+ crate::tensor::backend::HostReadback,
K: ConstDType,
I: IntoIterator<Item = &'a B::Var<K>>,
{
let mut found_nan_or_inf = false;
let mut present = alloc::vec::Vec::new();
for var in vars {
let tensor = B::var_as_tensor::<K>(var)?;
if let Some(grad) = B::get_grad::<K>(&tensor, grads.as_backend())? {
let float_vals = B::float_to_vec1::<K>(&grad)?;
if float_vals.iter().any(|v| !v.is_finite()) {
found_nan_or_inf = true;
break;
}
present.push((tensor, grad));
}
}
let current_scale = self.scale();
self.update(found_nan_or_inf);
if found_nan_or_inf {
return Ok(false);
}
if (current_scale - 1.0).abs() > f32::EPSILON {
let inv_scale = 1.0 / (current_scale as f64);
for (tensor, grad) in present {
let unscaled = B::optimizer_mul_scalar(&grad, inv_scale)?;
B::set_grad::<K>(&tensor, grads.as_backend_mut(), unscaled)?;
}
}
Ok(true)
}
}
#[derive(Debug, Default, Clone, Copy, PartialEq)]
pub enum LossScaling {
#[default]
None,
Static(f32),
Dynamic {
initial_scale: f32,
growth_factor: f32,
backoff_factor: f32,
growth_interval: usize,
},
}
impl LossScaling {
#[must_use]
pub const fn static_scale(scale: f32) -> Self {
Self::Static(scale)
}
#[must_use]
pub const fn dynamic_default() -> Self {
Self::Dynamic {
initial_scale: 65536.0,
growth_factor: 2.0,
backoff_factor: 0.5,
growth_interval: 2000,
}
}
#[must_use]
pub const fn dynamic(
initial_scale: f32,
growth_factor: f32,
backoff_factor: f32,
growth_interval: usize,
) -> Self {
Self::Dynamic {
initial_scale,
growth_factor,
backoff_factor,
growth_interval,
}
}
#[must_use]
pub fn initial_scale(&self) -> f32 {
match *self {
Self::None => 1.0,
Self::Static(scale) => scale,
Self::Dynamic { initial_scale, .. } => initial_scale,
}
}
}
impl core::hash::Hash for LossScaling {
fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
core::mem::discriminant(self).hash(state);
match *self {
Self::None => {}
Self::Static(s) => s.to_bits().hash(state),
Self::Dynamic {
initial_scale,
growth_factor,
backoff_factor,
growth_interval,
} => {
initial_scale.to_bits().hash(state);
growth_factor.to_bits().hash(state);
backoff_factor.to_bits().hash(state);
growth_interval.hash(state);
}
}
}
}
impl Eq for LossScaling {}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub enum PrecisionChoice {
#[default]
Native,
Exact(DTypeDescriptor),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct RuntimePrecisionPolicy {
pub(crate) generated: DTypeDescriptor,
pub(crate) parameter: DTypeDescriptor,
pub(crate) active_dtype: Option<DTypeDescriptor>,
pub(crate) compute: PrecisionChoice,
pub(crate) accumulator: PrecisionChoice,
pub(crate) loss_scaling: LossScaling,
}
impl core::default::Default for RuntimePrecisionPolicy {
fn default() -> Self {
Self::fp32()
}
}
impl RuntimePrecisionPolicy {
#[must_use]
pub const fn fp32() -> Self {
Self {
generated: <f32 as ConstDType>::DESCRIPTOR,
parameter: <f32 as ConstDType>::DESCRIPTOR,
active_dtype: None,
compute: PrecisionChoice::Native,
accumulator: PrecisionChoice::Native,
loss_scaling: LossScaling::None,
}
}
#[must_use]
pub const fn fp64() -> Self {
Self::exact_dtype(<f64 as ConstDType>::DESCRIPTOR)
}
#[must_use]
pub const fn exact_dtype(dtype: DTypeDescriptor) -> Self {
Self {
generated: dtype,
parameter: dtype,
active_dtype: Some(dtype),
compute: PrecisionChoice::Exact(dtype),
accumulator: PrecisionChoice::Exact(dtype),
loss_scaling: LossScaling::None,
}
}
pub fn exact<K: ConstDType + FloatDType>() -> Self {
Self::exact_dtype(K::DESCRIPTOR)
}
#[must_use]
pub const fn mixed_f16() -> Self {
Self {
generated: <f16 as ConstDType>::DESCRIPTOR,
parameter: <f16 as ConstDType>::DESCRIPTOR,
active_dtype: Some(<f16 as ConstDType>::DESCRIPTOR),
compute: PrecisionChoice::Native,
accumulator: PrecisionChoice::Exact(<f32 as ConstDType>::DESCRIPTOR),
loss_scaling: LossScaling::dynamic_default(),
}
}
#[must_use]
pub const fn mixed_bf16() -> Self {
Self {
generated: <bf16 as ConstDType>::DESCRIPTOR,
parameter: <bf16 as ConstDType>::DESCRIPTOR,
active_dtype: Some(<bf16 as ConstDType>::DESCRIPTOR),
compute: PrecisionChoice::Native,
accumulator: PrecisionChoice::Exact(<f32 as ConstDType>::DESCRIPTOR),
loss_scaling: LossScaling::None,
}
}
#[must_use]
pub const fn generated(&self) -> DTypeDescriptor {
self.generated
}
#[must_use]
pub const fn parameter(&self) -> DTypeDescriptor {
self.parameter
}
pub const fn active_dtype(&self) -> Option<DTypeDescriptor> {
self.active_dtype
}
#[must_use]
pub const fn compute(&self) -> PrecisionChoice {
self.compute
}
#[must_use]
pub const fn accumulator(&self) -> PrecisionChoice {
self.accumulator
}
#[must_use]
pub const fn loss_scaling(&self) -> LossScaling {
self.loss_scaling
}
#[must_use]
pub const fn with_compute(mut self, compute: PrecisionChoice) -> Self {
self.compute = compute;
self
}
#[must_use]
pub const fn with_accumulator(mut self, accumulator: PrecisionChoice) -> Self {
self.accumulator = accumulator;
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct PrecisionRequest {
pub operation: OperationKind,
pub storage: DTypeDescriptor,
pub output: DTypeDescriptor,
pub layout: LayoutClass,
pub rank: usize,
pub training: bool,
pub math_mode: MathMode,
}
impl PrecisionRequest {
#[must_use]
pub const fn new(
operation: OperationKind,
storage: DTypeDescriptor,
output: DTypeDescriptor,
layout: LayoutClass,
rank: usize,
training: bool,
math_mode: MathMode,
) -> Self {
Self {
operation,
storage,
output,
layout,
rank,
training,
math_mode,
}
}
}
impl core::default::Default for PrecisionRequest {
fn default() -> Self {
Self {
operation: OperationKind::Pointwise,
storage: <f32 as ConstDType>::DESCRIPTOR,
output: <f32 as ConstDType>::DESCRIPTOR,
layout: LayoutClass::Contiguous,
rank: 1,
training: false,
math_mode: MathMode::Fast,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ResolvedPrecision {
pub storage: DTypeDescriptor,
pub compute: DTypeDescriptor,
pub accumulator: DTypeDescriptor,
pub output: DTypeDescriptor,
pub loss_scaling: LossScaling,
}
impl ResolvedPrecision {
#[must_use]
pub const fn new(
storage: DTypeDescriptor,
compute: DTypeDescriptor,
accumulator: DTypeDescriptor,
output: DTypeDescriptor,
loss_scaling: LossScaling,
) -> Self {
Self {
storage,
compute,
accumulator,
output,
loss_scaling,
}
}
#[must_use]
pub const fn storage(&self) -> DTypeDescriptor {
self.storage
}
#[must_use]
pub const fn compute(&self) -> DTypeDescriptor {
self.compute
}
#[must_use]
pub const fn accumulator(&self) -> DTypeDescriptor {
self.accumulator
}
#[must_use]
pub const fn output(&self) -> DTypeDescriptor {
self.output
}
#[must_use]
pub const fn loss_scaling(&self) -> LossScaling {
self.loss_scaling
}
}
pub trait PrecisionCapabilities {
fn native_precision(&self, request: &PrecisionRequest) -> Result<ResolvedPrecision>;
}
pub fn resolve_precision<B>(
backend: &B,
policy: RuntimePrecisionPolicy,
request: &PrecisionRequest,
) -> Result<ResolvedPrecision>
where
B: PrecisionCapabilities + ?Sized,
{
let native = backend.native_precision(request)?;
if let Some(active) = policy.active_dtype
&& request.storage != active
{
return Ok(ResolvedPrecision {
storage: request.storage,
compute: native.compute,
accumulator: native.accumulator,
output: request.output,
loss_scaling: LossScaling::None,
});
}
let compute = match policy.compute {
PrecisionChoice::Native => native.compute,
PrecisionChoice::Exact(k) => {
if native.compute == k {
k
} else {
return Err(Error::UnsupportedPrecision {
operation: request.operation,
storage: request.storage,
requested: k,
role: PrecisionRole::Compute,
backend: core::any::type_name::<B>(),
});
}
}
};
let accumulator = match policy.accumulator {
PrecisionChoice::Native => native.accumulator,
PrecisionChoice::Exact(k) => {
if native.accumulator == k {
k
} else {
return Err(Error::UnsupportedPrecision {
operation: request.operation,
storage: request.storage,
requested: k,
role: PrecisionRole::Accumulator,
backend: core::any::type_name::<B>(),
});
}
}
};
let loss_scaling = match policy.active_dtype {
Some(active) if request.storage == active => policy.loss_scaling,
None => policy.loss_scaling,
Some(_) => LossScaling::None,
};
Ok(ResolvedPrecision {
storage: request.storage,
compute,
accumulator,
output: request.output,
loss_scaling,
})
}
pub trait PrecisionSpec:
'static + Send + Sync + Copy + core::fmt::Debug + Eq + PartialEq + core::hash::Hash
{
type Field: Clone + Send + Sync + 'static + core::fmt::Debug;
type GeneratedDType: DType;
type ParameterDType: DType;
fn init_field(&self) -> Self::Field;
fn generated_dtype_field(field: &Self::Field) -> <Self::GeneratedDType as DType>::Field;
fn parameter_dtype_field(field: &Self::Field) -> <Self::ParameterDType as DType>::Field;
fn runtime_policy(field: &Self::Field) -> RuntimePrecisionPolicy;
}
#[allow(clippy::module_inception)]
pub mod precision {
use super::*;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)]
pub struct Default;
impl PrecisionSpec for Default {
type Field = ();
type GeneratedDType = f32;
type ParameterDType = f32;
fn init_field(&self) -> Self::Field {}
fn generated_dtype_field(_field: &Self::Field) -> <Self::GeneratedDType as DType>::Field {
<Self::GeneratedDType as DType>::init(())
}
fn parameter_dtype_field(_field: &Self::Field) -> <Self::ParameterDType as DType>::Field {
<Self::ParameterDType as DType>::init(())
}
fn runtime_policy(_field: &Self::Field) -> RuntimePrecisionPolicy {
RuntimePrecisionPolicy::fp32()
}
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)]
pub struct Bf16Mixed;
pub type MixedBf16 = Bf16Mixed;
impl PrecisionSpec for Bf16Mixed {
type Field = ();
type GeneratedDType = bf16;
type ParameterDType = bf16;
fn init_field(&self) -> Self::Field {}
fn generated_dtype_field(_field: &Self::Field) -> <Self::GeneratedDType as DType>::Field {
<Self::GeneratedDType as DType>::init(())
}
fn parameter_dtype_field(_field: &Self::Field) -> <Self::ParameterDType as DType>::Field {
<Self::ParameterDType as DType>::init(())
}
fn runtime_policy(_field: &Self::Field) -> RuntimePrecisionPolicy {
RuntimePrecisionPolicy::mixed_bf16()
}
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)]
pub struct F16Mixed;
pub type MixedF16 = F16Mixed;
impl PrecisionSpec for F16Mixed {
type Field = ();
type GeneratedDType = f16;
type ParameterDType = f16;
fn init_field(&self) -> Self::Field {}
fn generated_dtype_field(_field: &Self::Field) -> <Self::GeneratedDType as DType>::Field {
<Self::GeneratedDType as DType>::init(())
}
fn parameter_dtype_field(_field: &Self::Field) -> <Self::ParameterDType as DType>::Field {
<Self::ParameterDType as DType>::init(())
}
fn runtime_policy(_field: &Self::Field) -> RuntimePrecisionPolicy {
RuntimePrecisionPolicy::mixed_f16()
}
}
pub struct Exact<K: BuiltinDType + FloatDType>(core::marker::PhantomData<K>);
impl<K: BuiltinDType + FloatDType> Clone for Exact<K> {
fn clone(&self) -> Self {
*self
}
}
impl<K: BuiltinDType + FloatDType> Copy for Exact<K> {}
impl<K: BuiltinDType + FloatDType> core::fmt::Debug for Exact<K> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("Exact").finish()
}
}
impl<K: BuiltinDType + FloatDType> core::default::Default for Exact<K> {
fn default() -> Self {
Self::new()
}
}
impl<K: BuiltinDType + FloatDType> PartialEq for Exact<K> {
fn eq(&self, _other: &Self) -> bool {
true
}
}
impl<K: BuiltinDType + FloatDType> Eq for Exact<K> {}
impl<K: BuiltinDType + FloatDType> core::hash::Hash for Exact<K> {
fn hash<H: core::hash::Hasher>(&self, _state: &mut H) {}
}
impl<K: BuiltinDType + FloatDType> Exact<K> {
#[must_use]
pub const fn new() -> Self {
Self(core::marker::PhantomData)
}
}
impl<K: BuiltinDType + FloatDType> PrecisionSpec for Exact<K> {
type Field = ();
type GeneratedDType = K;
type ParameterDType = K;
fn init_field(&self) -> Self::Field {}
fn generated_dtype_field(_field: &Self::Field) -> <K as DType>::Field {
K::init(())
}
fn parameter_dtype_field(_field: &Self::Field) -> <K as DType>::Field {
K::init(())
}
fn runtime_policy(_field: &Self::Field) -> RuntimePrecisionPolicy {
RuntimePrecisionPolicy::exact::<K>()
}
}
}
pub use precision::{Bf16Mixed, Default, Exact, F16Mixed, MixedBf16, MixedF16};
impl PrecisionSpec for Dyn {
type Field = RuntimePrecisionPolicy;
type GeneratedDType = Dyn;
type ParameterDType = Dyn;
fn init_field(&self) -> Self::Field {
RuntimePrecisionPolicy::default()
}
fn generated_dtype_field(field: &Self::Field) -> DTypeDescriptor {
field.generated
}
fn parameter_dtype_field(field: &Self::Field) -> DTypeDescriptor {
field.parameter
}
fn runtime_policy(field: &Self::Field) -> RuntimePrecisionPolicy {
*field
}
}