#[cfg(feature = "gpu")]
use crate::gpu::{GpuContext, GpuError};
use crate::Tensor;
use half::{bf16, f16};
use serde::{Deserialize, Serialize};
use std::fmt;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum AmpFormat {
#[default]
Fp16,
Bf16,
}
impl AmpFormat {
pub fn as_str(self) -> &'static str {
match self {
Self::Fp16 => "fp16",
Self::Bf16 => "bf16",
}
}
pub fn parse(value: &str) -> Result<Self, AmpError> {
match value.trim().to_ascii_lowercase().as_str() {
"fp16" | "f16" | "half" => Ok(Self::Fp16),
"bf16" | "bfloat16" => Ok(Self::Bf16),
other => Err(AmpError(format!(
"unknown amp format '{other}'; expected fp16 or bf16"
))),
}
}
}
fn default_initial_loss_scale() -> f32 {
65_536.0
}
fn default_growth_factor() -> f32 {
2.0
}
fn default_backoff_factor() -> f32 {
0.5
}
fn default_growth_interval() -> usize {
2_000
}
fn default_min_loss_scale() -> f32 {
1.0
}
fn default_max_loss_scale() -> f32 {
16_777_216.0
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct MixedPrecisionConfig {
#[serde(default)]
pub enabled: bool,
#[serde(default)]
pub format: AmpFormat,
#[serde(default = "default_initial_loss_scale")]
pub initial_loss_scale: f32,
#[serde(default = "default_growth_factor")]
pub growth_factor: f32,
#[serde(default = "default_backoff_factor")]
pub backoff_factor: f32,
#[serde(default = "default_growth_interval")]
pub growth_interval: usize,
#[serde(default = "default_min_loss_scale")]
pub min_loss_scale: f32,
#[serde(default = "default_max_loss_scale")]
pub max_loss_scale: f32,
}
impl Default for MixedPrecisionConfig {
fn default() -> Self {
Self {
enabled: false,
format: AmpFormat::Fp16,
initial_loss_scale: default_initial_loss_scale(),
growth_factor: default_growth_factor(),
backoff_factor: default_backoff_factor(),
growth_interval: default_growth_interval(),
min_loss_scale: default_min_loss_scale(),
max_loss_scale: default_max_loss_scale(),
}
}
}
impl MixedPrecisionConfig {
pub fn disabled() -> Self {
Self::default()
}
pub fn enabled_fp16() -> Self {
Self {
enabled: true,
format: AmpFormat::Fp16,
..Self::default()
}
}
pub fn validate(&self) -> Result<(), AmpError> {
if !self.enabled {
return Ok(());
}
for (name, value) in [
("initial_loss_scale", self.initial_loss_scale),
("growth_factor", self.growth_factor),
("backoff_factor", self.backoff_factor),
("min_loss_scale", self.min_loss_scale),
("max_loss_scale", self.max_loss_scale),
] {
if !value.is_finite() || value <= 0.0 {
return Err(AmpError(format!(
"{name} must be finite and positive"
)));
}
}
if self.growth_factor <= 1.0 {
return Err(AmpError("growth_factor must be greater than 1".into()));
}
if !(0.0..1.0).contains(&self.backoff_factor) {
return Err(AmpError("backoff_factor must be in (0, 1)".into()));
}
if self.growth_interval == 0 {
return Err(AmpError("growth_interval must be positive".into()));
}
if self.min_loss_scale > self.max_loss_scale {
return Err(AmpError(
"min_loss_scale must not exceed max_loss_scale".into(),
));
}
if self.initial_loss_scale < self.min_loss_scale
|| self.initial_loss_scale > self.max_loss_scale
{
return Err(AmpError(
"initial_loss_scale must be within [min_loss_scale, max_loss_scale]".into(),
));
}
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct LossScalerState {
pub scale: f32,
pub consecutive_successful_steps: usize,
}
#[derive(Clone, Debug)]
pub struct DynamicLossScaler {
scale: f32,
growth_factor: f32,
backoff_factor: f32,
growth_interval: usize,
consecutive_successful_steps: usize,
min_scale: f32,
max_scale: f32,
}
impl DynamicLossScaler {
pub fn from_config(config: &MixedPrecisionConfig) -> Self {
Self {
scale: config.initial_loss_scale,
growth_factor: config.growth_factor,
backoff_factor: config.backoff_factor,
growth_interval: config.growth_interval,
consecutive_successful_steps: 0,
min_scale: config.min_loss_scale,
max_scale: config.max_loss_scale,
}
}
pub fn from_state(config: &MixedPrecisionConfig, state: &LossScalerState) -> Self {
let mut scaler = Self::from_config(config);
scaler.scale = state.scale.clamp(config.min_loss_scale, config.max_loss_scale);
scaler.consecutive_successful_steps = state.consecutive_successful_steps;
scaler
}
pub fn scale(&self) -> f32 {
self.scale
}
pub fn state(&self) -> LossScalerState {
LossScalerState {
scale: self.scale,
consecutive_successful_steps: self.consecutive_successful_steps,
}
}
pub fn update(&mut self, found_inf: bool) {
if found_inf {
self.scale = (self.scale * self.backoff_factor).max(self.min_scale);
self.consecutive_successful_steps = 0;
return;
}
self.consecutive_successful_steps += 1;
if self.consecutive_successful_steps >= self.growth_interval {
self.scale = (self.scale * self.growth_factor).min(self.max_scale);
self.consecutive_successful_steps = 0;
}
}
}
#[derive(Debug)]
pub struct AmpError(pub String);
impl fmt::Display for AmpError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(formatter, "mixed-precision error: {}", self.0)
}
}
impl std::error::Error for AmpError {}
pub fn cast_storage(values: &[f32], format: AmpFormat) -> Vec<f32> {
match format {
AmpFormat::Fp16 => values
.iter()
.map(|value| f16::from_f32(*value).to_f32())
.collect(),
AmpFormat::Bf16 => values
.iter()
.map(|value| bf16::from_f32(*value).to_f32())
.collect(),
}
}
fn overflows_format(value: f32, format: AmpFormat) -> bool {
if !value.is_finite() {
return true;
}
match format {
AmpFormat::Fp16 => {
let cast = f16::from_f32(value);
cast.is_nan() || cast.is_infinite()
}
AmpFormat::Bf16 => {
let cast = bf16::from_f32(value);
cast.is_nan() || cast.is_infinite()
}
}
}
pub fn gradients_overflow(gradients: &[f32], format: AmpFormat) -> bool {
gradients.iter().any(|value| overflows_format(*value, format))
}
pub fn parameters_overflow(parameters: &[Tensor], format: AmpFormat) -> bool {
parameters.iter().any(|parameter| {
parameter
.grad()
.is_some_and(|gradient| gradients_overflow(&gradient, format))
})
}
#[cfg(feature = "gpu")]
pub fn parameters_overflow_gpu(
parameters: &[Tensor],
format: AmpFormat,
_gpu: &GpuContext,
) -> Result<bool, GpuError> {
for parameter in parameters {
let Some(gradient) = parameter.gpu_grad() else {
continue;
};
let values = gradient.to_vec()?;
if gradients_overflow(&values, format) {
return Ok(true);
}
}
Ok(false)
}
#[cfg(not(feature = "gpu"))]
pub fn parameters_overflow_gpu(
_parameters: &[Tensor],
_format: AmpFormat,
_gpu: &crate::gpu::GpuContext,
) -> Result<bool, crate::gpu::GpuError> {
Err(crate::gpu::GpuError::Initialization(
"GPU support is not compiled in; rebuild with `--features gpu`".into(),
))
}
pub fn scale_parameter_grads(parameters: &[Tensor], scale: f32) {
if scale == 1.0 {
return;
}
for parameter in parameters {
parameter.scale_grad(scale);
}
}
#[cfg(test)]
mod tests {
use super::{
cast_storage, gradients_overflow, AmpFormat, DynamicLossScaler, MixedPrecisionConfig,
};
#[test]
fn scaler_backs_off_on_overflow_and_grows_after_interval() {
let config = MixedPrecisionConfig {
enabled: true,
initial_loss_scale: 1024.0,
growth_factor: 2.0,
backoff_factor: 0.5,
growth_interval: 2,
min_loss_scale: 1.0,
max_loss_scale: 4096.0,
..MixedPrecisionConfig::default()
};
let mut scaler = DynamicLossScaler::from_config(&config);
assert_eq!(scaler.scale(), 1024.0);
scaler.update(true);
assert_eq!(scaler.scale(), 512.0);
scaler.update(false);
assert_eq!(scaler.scale(), 512.0);
scaler.update(false);
assert_eq!(scaler.scale(), 1024.0);
}
#[test]
fn fp16_cast_detects_overflow_and_preserves_fp32_masters() {
let master = vec![1.0 + 1e-4, 2.0, 3.0];
let working = cast_storage(&master, AmpFormat::Fp16);
assert_ne!(working[0], master[0]);
assert!(!gradients_overflow(&[1.0, -2.0], AmpFormat::Fp16));
assert!(gradients_overflow(&[f32::INFINITY], AmpFormat::Fp16));
assert!(gradients_overflow(&[7.0e5], AmpFormat::Fp16));
assert!(!gradients_overflow(&[7.0e5], AmpFormat::Bf16));
}
#[test]
fn mixed_precision_config_rejects_invalid_scales() {
let mut config = MixedPrecisionConfig::enabled_fp16();
config.backoff_factor = 1.5;
assert!(config.validate().is_err());
config.backoff_factor = 0.5;
config.growth_interval = 0;
assert!(config.validate().is_err());
}
}