use rayon::prelude::*;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub enum Optimizer {
Sgd {
learning_rate: f32,
},
Adam {
learning_rate: f32,
beta1: f32,
beta2: f32,
epsilon: f32,
#[serde(default)]
weight_decay: f32,
},
Lion {
learning_rate: f32,
beta1: f32,
beta2: f32,
weight_decay: f32,
},
}
pub(crate) fn sign(value: f32) -> f32 {
if value > 0.0 {
1.0
} else if value < 0.0 {
-1.0
} else {
0.0
}
}
impl Optimizer {
pub fn sgd(learning_rate: f32) -> Self {
Self::Sgd { learning_rate }
}
pub fn adam(learning_rate: f32) -> Self {
Self::Adam {
learning_rate,
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
weight_decay: 0.0,
}
}
pub fn adam_with_weight_decay(learning_rate: f32, weight_decay: f32) -> Self {
Self::Adam {
learning_rate,
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
weight_decay,
}
}
pub fn lion(learning_rate: f32) -> Self {
Self::Lion {
learning_rate,
beta1: 0.9,
beta2: 0.99,
weight_decay: 0.0,
}
}
pub fn lion_with_weight_decay(learning_rate: f32, weight_decay: f32) -> Self {
Self::Lion {
learning_rate,
beta1: 0.9,
beta2: 0.99,
weight_decay,
}
}
pub fn set_learning_rate(&mut self, rate: f32) {
match self {
Self::Sgd { learning_rate }
| Self::Adam { learning_rate, .. }
| Self::Lion { learning_rate, .. } => *learning_rate = rate,
}
}
pub fn learning_rate(&self) -> f32 {
match self {
Self::Sgd { learning_rate }
| Self::Adam { learning_rate, .. }
| Self::Lion { learning_rate, .. } => *learning_rate,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Schedule {
peak: f32,
warmup: usize,
steps: usize,
floor: f32,
}
impl Schedule {
pub fn warmup_cosine(peak: f32, warmup: usize, steps: usize) -> Self {
Self {
peak,
warmup,
steps,
floor: 0.1,
}
}
pub fn floor(mut self, fraction: f32) -> Self {
self.floor = fraction;
self
}
pub fn rate(&self, step: usize) -> f32 {
if step < self.warmup {
return self.peak * (step + 1) as f32 / self.warmup as f32;
}
let decay = self.steps.saturating_sub(self.warmup);
if decay == 0 {
return self.peak * self.floor;
}
let progress = ((step - self.warmup) as f32 / decay as f32).min(1.0);
let cosine = 0.5 * (1.0 + (std::f32::consts::PI * progress).cos());
self.peak * (self.floor + (1.0 - self.floor) * cosine)
}
}
use crate::param::Param;
pub fn grad_norm(params: &mut [&mut Param]) -> Result<f32, crate::network::NetworkError> {
let mut total = 0.0;
for param in params.iter() {
total += param.grad_sum_squares()?;
}
Ok(total.sqrt() as f32)
}
pub fn apply_step(params: &mut [&mut Param], optimizer: &Optimizer, step: usize, scale: f32) {
if params.iter().any(|param| param.is_on_device()) {
for param in params.iter_mut() {
param.step(optimizer, step, scale);
}
return;
}
params
.par_iter_mut()
.for_each(|param| param.step(optimizer, step, scale));
}
pub fn step_clipped(
params: &mut [&mut Param],
optimizer: &Optimizer,
step: usize,
scale: f32,
max_norm: f32,
) -> Result<f32, crate::network::NetworkError> {
let norm = grad_norm(params)?;
let clip = match norm > max_norm && norm > 0.0 {
true => max_norm / norm,
false => 1.0,
};
apply_step(params, optimizer, step, scale * clip);
Ok(norm)
}
pub fn zero_grad(params: &mut [&mut Param]) {
if params.iter().any(|param| param.is_on_device()) {
for param in params.iter_mut() {
param.zero_grad();
}
return;
}
params.par_iter_mut().for_each(|param| param.zero_grad());
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn warmup_rises_to_the_peak_and_the_cosine_falls_to_the_floor() {
let schedule = Schedule::warmup_cosine(1.0, 10, 110);
assert!((schedule.rate(0) - 0.1).abs() < 1e-6);
assert!((schedule.rate(9) - 1.0).abs() < 1e-6);
assert!((schedule.rate(10) - 1.0).abs() < 1e-6);
assert!((schedule.rate(60) - 0.55).abs() < 1e-3); assert!((schedule.rate(110) - 0.1).abs() < 1e-6);
assert!((schedule.rate(10_000) - 0.1).abs() < 1e-6);
for step in 10..110 {
assert!(schedule.rate(step) >= schedule.rate(step + 1));
}
}
#[test]
fn the_floor_is_adjustable_and_a_schedule_with_no_decay_is_flat() {
assert!(Schedule::warmup_cosine(1.0, 0, 100).floor(0.0).rate(100) < 1e-6);
assert!((Schedule::warmup_cosine(1.0, 100, 100).rate(100) - 0.1).abs() < 1e-6);
}
#[test]
fn setting_the_rate_keeps_the_rest_of_the_optimizer() {
let mut optimizer = Optimizer::adam_with_weight_decay(1e-3, 0.1);
optimizer.set_learning_rate(2e-4);
assert_eq!(optimizer.learning_rate(), 2e-4);
assert_eq!(optimizer, Optimizer::adam_with_weight_decay(2e-4, 0.1));
}
}