use rayon::prelude::*;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub enum Optimizer {
Sgd {
learning_rate: f32,
},
Adam {
learning_rate: f32,
beta1: f32,
beta2: f32,
epsilon: f32,
#[serde(default)]
weight_decay: f32,
},
Lion {
learning_rate: f32,
beta1: f32,
beta2: f32,
weight_decay: f32,
},
}
pub(crate) fn sign(value: f32) -> f32 {
if value > 0.0 {
1.0
} else if value < 0.0 {
-1.0
} else {
0.0
}
}
impl Optimizer {
pub fn sgd(learning_rate: f32) -> Self {
Self::Sgd { learning_rate }
}
pub fn adam(learning_rate: f32) -> Self {
Self::Adam {
learning_rate,
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
weight_decay: 0.0,
}
}
pub fn adam_with_weight_decay(learning_rate: f32, weight_decay: f32) -> Self {
Self::Adam {
learning_rate,
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
weight_decay,
}
}
pub fn lion(learning_rate: f32) -> Self {
Self::Lion {
learning_rate,
beta1: 0.9,
beta2: 0.99,
weight_decay: 0.0,
}
}
pub fn lion_with_weight_decay(learning_rate: f32, weight_decay: f32) -> Self {
Self::Lion {
learning_rate,
beta1: 0.9,
beta2: 0.99,
weight_decay,
}
}
pub fn set_learning_rate(&mut self, rate: f32) {
match self {
Self::Sgd { learning_rate }
| Self::Adam { learning_rate, .. }
| Self::Lion { learning_rate, .. } => *learning_rate = rate,
}
}
pub(crate) fn without_weight_decay(&self) -> Self {
let mut optimizer = self.clone();
if let Self::Adam { weight_decay, .. } | Self::Lion { weight_decay, .. } = &mut optimizer {
*weight_decay = 0.0;
}
optimizer
}
pub fn learning_rate(&self) -> f32 {
match self {
Self::Sgd { learning_rate }
| Self::Adam { learning_rate, .. }
| Self::Lion { learning_rate, .. } => *learning_rate,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Schedule {
peak: f32,
warmup: usize,
steps: usize,
floor: f32,
linear_tail: Option<f32>,
}
impl Schedule {
pub fn warmup_cosine(peak: f32, warmup: usize, steps: usize) -> Self {
Self {
peak,
warmup,
steps,
floor: 0.1,
linear_tail: None,
}
}
pub fn warmup_stable_decay(
peak: f32,
warmup: usize,
steps: usize,
decay_fraction: f32,
) -> Self {
Self {
linear_tail: Some(decay_fraction.clamp(0.0, 1.0)),
..Self::warmup_cosine(peak, warmup, steps)
}
}
pub fn floor(mut self, fraction: f32) -> Self {
self.floor = fraction;
self
}
pub fn rate(&self, step: usize) -> f32 {
if step < self.warmup {
return self.peak * (step + 1) as f32 / self.warmup as f32;
}
if let Some(fraction) = self.linear_tail {
let decay = (self.steps as f32 * fraction).round() as usize;
let start = self.steps.saturating_sub(decay).max(self.warmup);
if step < start {
return self.peak;
}
let progress = match self.steps > start {
true => ((step - start) as f32 / (self.steps - start) as f32).min(1.0),
false => 1.0,
};
return self.peak * (1.0 - (1.0 - self.floor) * progress);
}
let decay = self.steps.saturating_sub(self.warmup);
if decay == 0 {
return self.peak * self.floor;
}
let progress = ((step - self.warmup) as f32 / decay as f32).min(1.0);
let cosine = 0.5 * (1.0 + (std::f32::consts::PI * progress).cos());
self.peak * (self.floor + (1.0 - self.floor) * cosine)
}
}
use crate::param::Param;
pub fn grad_norm(params: &mut [&mut Param]) -> Result<f32, crate::network::NetworkError> {
let mut total = 0.0;
for param in params.iter() {
total += param.grad_sum_squares()?;
}
Ok(total.sqrt() as f32)
}
pub fn apply_step(params: &mut [&mut Param], optimizer: &Optimizer, step: usize, scale: f32) {
if params.iter().any(|param| param.is_on_device()) {
for param in params.iter_mut() {
param.step(optimizer, step, scale);
}
return;
}
params
.par_iter_mut()
.for_each(|param| param.step(optimizer, step, scale));
}
pub fn step_clipped(
params: &mut [&mut Param],
optimizer: &Optimizer,
step: usize,
scale: f32,
max_norm: f32,
) -> Result<f32, crate::network::NetworkError> {
let norm = grad_norm(params)?;
if !norm.is_finite() {
zero_grad(params);
return Ok(norm);
}
let clip = match norm > max_norm && norm > 0.0 {
true => max_norm / norm,
false => 1.0,
};
apply_step(params, optimizer, step, scale * clip);
Ok(norm)
}
pub fn zero_grad(params: &mut [&mut Param]) {
if params.iter().any(|param| param.is_on_device()) {
for param in params.iter_mut() {
param.zero_grad();
}
return;
}
params.par_iter_mut().for_each(|param| param.zero_grad());
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct SpikeGuard {
ratio: f32,
beta: f32,
warmup: usize,
mean: f32,
accepted: usize,
consecutive: usize,
}
impl SpikeGuard {
pub fn new(ratio: f32, beta: f32, warmup: usize) -> Self {
Self {
ratio,
beta,
warmup,
mean: 0.0,
accepted: 0,
consecutive: 0,
}
}
pub fn is_spike(&mut self, loss: f32) -> bool {
let spike =
!loss.is_finite() || (self.accepted >= self.warmup && loss > self.ratio * self.mean);
if spike {
self.consecutive += 1;
return true;
}
self.consecutive = 0;
self.accepted += 1;
let weight = (1.0 - self.beta) / (1.0 - self.beta.powi(self.accepted as i32));
self.mean += weight * (loss - self.mean);
false
}
pub fn consecutive(&self) -> usize {
self.consecutive
}
pub fn mean(&self) -> f32 {
self.mean
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_spike_is_skipped_without_moving_the_mean() {
let mut guard = SpikeGuard::new(2.0, 0.9, 3);
assert!(!guard.is_spike(10.0));
assert!((guard.mean() - 10.0).abs() < 1e-6);
assert!(!guard.is_spike(4.0));
assert!(!guard.is_spike(4.0));
let mean = guard.mean();
assert!(guard.is_spike(2.0 * mean + 0.1));
assert!(guard.is_spike(f32::INFINITY));
assert_eq!(guard.mean(), mean);
assert_eq!(guard.consecutive(), 2);
assert!(!guard.is_spike(2.0 * mean - 0.1));
assert_eq!(guard.consecutive(), 0);
}
#[test]
fn warmup_rises_to_the_peak_and_the_cosine_falls_to_the_floor() {
let schedule = Schedule::warmup_cosine(1.0, 10, 110);
assert!((schedule.rate(0) - 0.1).abs() < 1e-6);
assert!((schedule.rate(9) - 1.0).abs() < 1e-6);
assert!((schedule.rate(10) - 1.0).abs() < 1e-6);
assert!((schedule.rate(60) - 0.55).abs() < 1e-3); assert!((schedule.rate(110) - 0.1).abs() < 1e-6);
assert!((schedule.rate(10_000) - 0.1).abs() < 1e-6);
for step in 10..110 {
assert!(schedule.rate(step) >= schedule.rate(step + 1));
}
}
#[test]
fn the_floor_is_adjustable_and_a_schedule_with_no_decay_is_flat() {
assert!(Schedule::warmup_cosine(1.0, 0, 100).floor(0.0).rate(100) < 1e-6);
assert!((Schedule::warmup_cosine(1.0, 100, 100).rate(100) - 0.1).abs() < 1e-6);
}
#[test]
fn warmup_stable_decay_holds_the_peak_then_falls_linearly_to_the_floor() {
let schedule = Schedule::warmup_stable_decay(1.0, 10, 100, 0.2).floor(0.0);
assert!((schedule.rate(0) - 0.1).abs() < 1e-6);
for step in 9..=80 {
assert_eq!(schedule.rate(step), 1.0);
}
assert!((schedule.rate(90) - 0.5).abs() < 1e-6);
assert!(schedule.rate(100) < 1e-6);
assert!(schedule.rate(10_000) < 1e-6);
let longer = Schedule::warmup_stable_decay(1.0, 10, 200, 0.2).floor(0.0);
for step in 0..=80 {
assert_eq!(schedule.rate(step), longer.rate(step));
}
assert!((Schedule::warmup_stable_decay(1.0, 10, 100, 0.2).rate(100) - 0.1).abs() < 1e-6);
}
#[test]
fn setting_the_rate_keeps_the_rest_of_the_optimizer() {
let mut optimizer = Optimizer::adam_with_weight_decay(1e-3, 0.1);
optimizer.set_learning_rate(2e-4);
assert_eq!(optimizer.learning_rate(), 2e-4);
assert_eq!(optimizer, Optimizer::adam_with_weight_decay(2e-4, 0.1));
}
}