use candle_core::{Tensor, backprop::GradStore};
pub(crate) fn clip_gradients(
loss: &Tensor,
grad_store: &mut GradStore,
max_norm: f32,
) -> Result<f32, candle_core::Error> {
let mut grads = vec![];
let variable_ids = loss
.sorted_nodes()
.into_iter()
.filter(|node| node.is_variable())
.map(Tensor::id);
let mut norm_sqrs: Vec<f64> = vec![];
for id in variable_ids {
if let Some(grad) = grad_store.get_id(id) {
let norm_sq = grad
.sqr()?
.sum_all()?
.to_dtype(candle_core::DType::F64)?
.to_scalar::<f64>()?;
norm_sqrs.push(norm_sq);
grads.push((id, grad.clone()));
}
}
norm_sqrs.sort_by(f64::total_cmp);
let total_norm_sq = norm_sqrs.into_iter().sum::<f64>();
let total_norm = total_norm_sq.sqrt();
if total_norm > f64::from(max_norm) {
let scale = f64::from(max_norm) / (total_norm + 1e-6);
for (id, grad) in &grads {
let scale_t = Tensor::new(scale, grad.device())?.to_dtype(grad.dtype())?;
let clipped = grad.broadcast_mul(&scale_t)?;
grad_store.insert_id(*id, clipped);
}
}
Ok(total_norm as f32)
}
pub(crate) fn normalize_tensor(t: &Tensor) -> Result<Tensor, candle_core::Error> {
let mean = t.mean_all()?.broadcast_as(t.shape())?;
let diff = (t.clone() - mean.clone())?;
let n = t.elem_count().max(2) as f64;
let std = (diff.sqr()?.sum_all()? / (n - 1.0))?.sqrt()?;
let std_with_eps = (std + 1e-8)?.broadcast_as(t.shape())?;
diff / std_with_eps
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn gradient_clipping_matches_pytorch_epsilon() {
let device = candle_core::Device::Cpu;
let variable = candle_core::Var::from_vec(vec![3.0f32, 4.0], 2, &device).unwrap();
let loss = variable.sqr().unwrap().sum_all().unwrap();
let mut gradients = loss.backward().unwrap();
let original_norm = clip_gradients(&loss, &mut gradients, 5.0).unwrap();
let clipped = gradients
.get(variable.as_tensor())
.unwrap()
.to_vec1::<f32>()
.unwrap();
let expected_scale = 5.0 / (10.0 + 1e-6);
assert!((original_norm - 10.0).abs() < 1e-6);
assert!((clipped[0] - 6.0 * expected_scale).abs() < 1e-6);
assert!((clipped[1] - 8.0 * expected_scale).abs() < 1e-6);
}
#[test]
fn gradient_clipping_ignores_detached_operand_gradients() {
let device = candle_core::Device::Cpu;
let parameter = candle_core::Var::from_vec(vec![1_000.0f32, 1_000.0], 2, &device).unwrap();
let detached_operand = candle_core::Var::from_vec(vec![1.0f32, 1.0], 2, &device).unwrap();
let detached_operand = detached_operand.detach();
let loss = (parameter.as_tensor() * detached_operand)
.unwrap()
.sum_all()
.unwrap();
let mut gradients = loss.backward().unwrap();
let original_norm = clip_gradients(&loss, &mut gradients, 1.0).unwrap();
let clipped = gradients
.get(parameter.as_tensor())
.unwrap()
.to_vec1::<f32>()
.unwrap();
let expected_norm = 2.0f32.sqrt();
let expected_scale = 1.0 / (expected_norm + 1e-6);
assert!((original_norm - expected_norm).abs() < 1e-6);
assert!((clipped[0] - expected_scale).abs() < 1e-6);
assert!((clipped[1] - expected_scale).abs() < 1e-6);
}
}