use super::group::ParameterGroup;
use super::support::invalid_optimizer_config;
use super::traits::{OptimizerBackend, ValueClippingBackend};
use crate::autograd::Gradients;
use crate::err::Result;
use crate::tensor::backend::{AutogradBackend, HostReadback, VariableBackend};
use crate::tensor::dtype::ConstDType;
pub fn clip_grad_norm<B, K>(
params: &ParameterGroup<B, K>,
grads: &mut Gradients<B>,
max_norm: f64,
) -> Result<f64>
where
B: VariableBackend + AutogradBackend + OptimizerBackend<K> + HostReadback,
K: ConstDType,
{
const OPERATION: &str = "clip_grad_norm";
if !max_norm.is_finite() || max_norm <= 0.0 {
return Err(invalid_optimizer_config(
OPERATION,
"the maximum norm must be finite and greater than zero",
));
}
let mut squared_total = 0.0f64;
let mut present = alloc::vec::Vec::new();
for (_, var) in params.iter() {
let tensor = B::var_as_tensor::<K>(var)?;
let Some(grad) = B::get_grad::<K>(&tensor, grads.as_backend())? else {
continue;
};
for value in B::float_to_vec1::<K>(&grad)? {
squared_total += value * value;
}
present.push((tensor, grad));
}
let total_norm = squared_total.sqrt();
if !total_norm.is_finite() {
return Err(invalid_optimizer_config(
OPERATION,
"the gradient norm is not finite, so no finite rescale exists. Inspect the \
backward pass rather than clipping a NaN into range.",
));
}
if total_norm <= max_norm {
return Ok(total_norm);
}
let scale = max_norm / (total_norm + 1e-6);
for (tensor, grad) in present {
let scaled = B::optimizer_mul_scalar(&grad, scale)?;
B::set_grad::<K>(&tensor, grads.as_backend_mut(), scaled)?;
}
Ok(total_norm)
}
pub fn clip_grad_value<B, K>(
params: &ParameterGroup<B, K>,
grads: &mut Gradients<B>,
clip_value: f64,
) -> Result<()>
where
B: VariableBackend + AutogradBackend + ValueClippingBackend<K>,
K: ConstDType,
{
const OPERATION: &str = "clip_grad_value";
if !clip_value.is_finite() || clip_value <= 0.0 {
return Err(invalid_optimizer_config(
OPERATION,
"the clip value must be finite and greater than zero",
));
}
for (_, var) in params.iter() {
let tensor = B::var_as_tensor::<K>(var)?;
let Some(grad) = B::get_grad::<K>(&tensor, grads.as_backend())? else {
continue;
};
let clamped = B::optimizer_clamp(&grad, -clip_value, clip_value)?;
B::set_grad::<K>(&tensor, grads.as_backend_mut(), clamped)?;
}
Ok(())
}