Skip to main content

rusting_brain/
optimizers.rs

1//! Parameter update rules.
2//!
3//! The optimizer is a value, not a trait object, and it is re-read from the
4//! model on every step. Assigning a fresh one with a new learning rate is all
5//! a warmup or cosine schedule needs to do.
6
7use rayon::prelude::*;
8use serde::{Deserialize, Serialize};
9
10#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
11pub enum Optimizer {
12    Sgd {
13        learning_rate: f32,
14    },
15    Adam {
16        learning_rate: f32,
17        beta1: f32,
18        beta2: f32,
19        epsilon: f32,
20        /// Decoupled (AdamW-style) weight decay applied to weights only, not
21        /// biases. `0.0` reproduces the original unregularized Adam.
22        #[serde(default)]
23        weight_decay: f32,
24    },
25    /// Lion: the update is the *sign* of a momentum-smoothed gradient, so every
26    /// parameter moves by exactly the learning rate.
27    ///
28    /// It keeps one moment where Adam keeps two, and the sign makes the step
29    /// size independent of the gradient's magnitude, so the learning rate wants
30    /// to be three to ten times smaller than an Adam rate and the weight decay
31    /// correspondingly larger. `Param` still allocates both moment buffers,
32    /// because the optimizer is a value that can be swapped between steps, so
33    /// the memory Lion saves elsewhere is not saved here.
34    ///
35    /// The CPU and CUDA paths run it. Metal has no kernel for it and reports
36    /// that rather than quietly taking an Adam step.
37    Lion {
38        learning_rate: f32,
39        beta1: f32,
40        beta2: f32,
41        /// Decoupled weight decay, as on [`Optimizer::Adam`].
42        weight_decay: f32,
43    },
44}
45
46/// The sign of `value`, and zero for zero.
47///
48/// `f32::signum` gives `1.0` for a positive zero, which would push a parameter
49/// whose smoothed gradient has vanished.
50pub(crate) fn sign(value: f32) -> f32 {
51    if value > 0.0 {
52        1.0
53    } else if value < 0.0 {
54        -1.0
55    } else {
56        0.0
57    }
58}
59
60impl Optimizer {
61    pub fn sgd(learning_rate: f32) -> Self {
62        Self::Sgd { learning_rate }
63    }
64
65    pub fn adam(learning_rate: f32) -> Self {
66        Self::Adam {
67            learning_rate,
68            beta1: 0.9,
69            beta2: 0.999,
70            epsilon: 1e-8,
71            weight_decay: 0.0,
72        }
73    }
74
75    pub fn adam_with_weight_decay(learning_rate: f32, weight_decay: f32) -> Self {
76        Self::Adam {
77            learning_rate,
78            beta1: 0.9,
79            beta2: 0.999,
80            epsilon: 1e-8,
81            weight_decay,
82        }
83    }
84
85    /// Lion at `learning_rate`, with the betas from the paper.
86    ///
87    /// Start from roughly a tenth of the Adam rate that works for the same
88    /// model: the update is a sign, so the rate alone sets the step size.
89    pub fn lion(learning_rate: f32) -> Self {
90        Self::Lion {
91            learning_rate,
92            beta1: 0.9,
93            beta2: 0.99,
94            weight_decay: 0.0,
95        }
96    }
97
98    pub fn lion_with_weight_decay(learning_rate: f32, weight_decay: f32) -> Self {
99        Self::Lion {
100            learning_rate,
101            beta1: 0.9,
102            beta2: 0.99,
103            weight_decay,
104        }
105    }
106
107    /// Replaces the learning rate and keeps everything else.
108    ///
109    /// This is what a schedule should assign. Building a fresh
110    /// `Optimizer::adam(rate)` each step silently drops the betas, the epsilon
111    /// and the weight decay back to their defaults.
112    pub fn set_learning_rate(&mut self, rate: f32) {
113        match self {
114            Self::Sgd { learning_rate }
115            | Self::Adam { learning_rate, .. }
116            | Self::Lion { learning_rate, .. } => *learning_rate = rate,
117        }
118    }
119
120    /// The same optimizer with its weight decay set to zero.
121    pub(crate) fn without_weight_decay(&self) -> Self {
122        let mut optimizer = self.clone();
123        if let Self::Adam { weight_decay, .. } | Self::Lion { weight_decay, .. } = &mut optimizer {
124            *weight_decay = 0.0;
125        }
126        optimizer
127    }
128
129    /// The current learning rate, which is worth logging next to the loss.
130    pub fn learning_rate(&self) -> f32 {
131        match self {
132            Self::Sgd { learning_rate }
133            | Self::Adam { learning_rate, .. }
134            | Self::Lion { learning_rate, .. } => *learning_rate,
135        }
136    }
137}
138
139/// Linear warmup, then a cosine decay to a fraction of the peak.
140///
141/// Full-size steps before Adam's moment estimates have settled can knock a
142/// model somewhere it takes a long time to leave; full-size steps at the end
143/// stop it settling at all. Every transformer run wants this curve, and it is
144/// arithmetic rather than state, so it is a plain value the caller reads once
145/// per step:
146///
147/// ```
148/// # use rusting_brain::{Optimizer, Schedule};
149/// let schedule = Schedule::warmup_cosine(3e-4, 2_000, 100_000);
150/// let mut optimizer = Optimizer::adam(3e-4);
151/// for step in 0..10 {
152///     optimizer.set_learning_rate(schedule.rate(step));
153///     // model.optimizer = optimizer.clone();
154/// }
155/// ```
156#[derive(Clone, Copy, Debug, PartialEq)]
157pub struct Schedule {
158    peak: f32,
159    warmup: usize,
160    steps: usize,
161    floor: f32,
162    /// `Some(fraction)` for warmup-stable-decay; `None` for cosine.
163    linear_tail: Option<f32>,
164}
165
166impl Schedule {
167    /// `warmup` steps rising to `peak`, then a cosine decay over the rest of
168    /// `steps`, ending at a tenth of the peak.
169    pub fn warmup_cosine(peak: f32, warmup: usize, steps: usize) -> Self {
170        Self {
171            peak,
172            warmup,
173            steps,
174            floor: 0.1,
175            linear_tail: None,
176        }
177    }
178
179    /// Warmup-stable-decay: `warmup` steps rising to `peak`, the peak held,
180    /// then a linear decay to the floor over the last `decay_fraction` of
181    /// `steps`.
182    ///
183    /// Every rate before the decay starts is the peak whatever `steps` is, so
184    /// a run can be extended by raising `steps` before its decay begins.
185    pub fn warmup_stable_decay(
186        peak: f32,
187        warmup: usize,
188        steps: usize,
189        decay_fraction: f32,
190    ) -> Self {
191        Self {
192            linear_tail: Some(decay_fraction.clamp(0.0, 1.0)),
193            ..Self::warmup_cosine(peak, warmup, steps)
194        }
195    }
196
197    /// The fraction of `peak` the decay ends at. The default is `0.1`, and
198    /// `0.0` decays to nothing.
199    pub fn floor(mut self, fraction: f32) -> Self {
200        self.floor = fraction;
201        self
202    }
203
204    /// The rate for `step`, counted from zero.
205    ///
206    /// Steps past the end of the schedule hold at the floor rather than going
207    /// negative, so a run extended beyond its planned length keeps training.
208    pub fn rate(&self, step: usize) -> f32 {
209        if step < self.warmup {
210            // `step + 1`, so the first step is not a forward and backward pass
211            // at a learning rate of zero, which changes nothing.
212            return self.peak * (step + 1) as f32 / self.warmup as f32;
213        }
214        if let Some(fraction) = self.linear_tail {
215            let decay = (self.steps as f32 * fraction).round() as usize;
216            let start = self.steps.saturating_sub(decay).max(self.warmup);
217            if step < start {
218                return self.peak;
219            }
220            let progress = match self.steps > start {
221                true => ((step - start) as f32 / (self.steps - start) as f32).min(1.0),
222                false => 1.0,
223            };
224            return self.peak * (1.0 - (1.0 - self.floor) * progress);
225        }
226        let decay = self.steps.saturating_sub(self.warmup);
227        if decay == 0 {
228            return self.peak * self.floor;
229        }
230        let progress = ((step - self.warmup) as f32 / decay as f32).min(1.0);
231        let cosine = 0.5 * (1.0 + (std::f32::consts::PI * progress).cos());
232        self.peak * (self.floor + (1.0 - self.floor) * cosine)
233    }
234}
235
236/// The training-step trio: a global gradient norm, the optimizer step, and the
237/// clipped step that combines them.
238///
239/// Every model in the crate accumulates gradients into a flat list of
240/// [`Param`]s and then applies exactly this sequence. Two copies of gradient
241/// clipping that drift apart is the kind of bug that shows up as a flat loss
242/// curve six hours into a run and nowhere else, so there is one copy and the
243/// models delegate to it.
244///
245/// ```
246/// # use rusting_brain::optimizers::{Optimizer, step_clipped};
247/// # use rusting_brain::param::Param;
248/// # use rusting_brain::matrix::Matrix;
249/// let mut weight = Param::zeros(1, 2);
250/// weight.grad = Matrix::from_vec(1, 2, vec![3.0, 4.0]);
251/// let norm = step_clipped(&mut [&mut weight], &Optimizer::adam(1e-2), 1, 1.0, 1.0)?;
252/// assert!((norm - 5.0).abs() < 1e-6);
253/// # Ok::<(), rusting_brain::NetworkError>(())
254/// ```
255use crate::param::Param;
256
257/// The L2 norm of the accumulated gradients, over every parameter at once.
258///
259/// Worth logging on its own: a run that is about to diverge shows it in this
260/// number one or two steps before the loss moves.
261pub fn grad_norm(params: &mut [&mut Param]) -> Result<f32, crate::network::NetworkError> {
262    let mut total = 0.0;
263    for param in params.iter() {
264        total += param.grad_sum_squares()?;
265    }
266    Ok(total.sqrt() as f32)
267}
268
269/// Applies the optimizer to every parameter.
270///
271/// `scale` divides the accumulated gradient, so a caller that ran several
272/// batches before stepping passes `1.0 / batches`. `step` counts from 1, as
273/// in [`Param::step`](crate::param::Param::step), which panics on 0.
274pub fn apply_step(params: &mut [&mut Param], optimizer: &Optimizer, step: usize, scale: f32) {
275    if params.iter().any(|param| param.is_on_device()) {
276        // One stream, one cuBLAS handle: the launches would serialize anyway,
277        // and a rayon pool around them only adds contention.
278        for param in params.iter_mut() {
279            param.step(optimizer, step, scale);
280        }
281        return;
282    }
283    params
284        .par_iter_mut()
285        .for_each(|param| param.step(optimizer, step, scale));
286}
287
288/// [`apply_step`] with the gradients clipped to a global norm of `max_norm`
289/// first, and the pre-clip norm returned for logging.
290///
291/// Clipping is a uniform rescale of every gradient, and `scale` already
292/// multiplies every gradient uniformly, so this folds the clip into that factor
293/// rather than rewriting the gradient buffers.
294///
295/// A norm that is NaN or infinite skips the step: the gradients are cleared
296/// and the parameters and optimizer moments are left as they were, because
297/// one non-finite update would poison both for the rest of the run. The norm
298/// is still returned, so the caller sees the skip.
299pub fn step_clipped(
300    params: &mut [&mut Param],
301    optimizer: &Optimizer,
302    step: usize,
303    scale: f32,
304    max_norm: f32,
305) -> Result<f32, crate::network::NetworkError> {
306    let norm = grad_norm(params)?;
307    if !norm.is_finite() {
308        zero_grad(params);
309        return Ok(norm);
310    }
311    let clip = match norm > max_norm && norm > 0.0 {
312        true => max_norm / norm,
313        false => 1.0,
314    };
315    apply_step(params, optimizer, step, scale * clip);
316    Ok(norm)
317}
318
319pub fn zero_grad(params: &mut [&mut Param]) {
320    if params.iter().any(|param| param.is_on_device()) {
321        for param in params.iter_mut() {
322            param.zero_grad();
323        }
324        return;
325    }
326    params.par_iter_mut().for_each(|param| param.zero_grad());
327}
328
329/// Flags a training step whose loss jumps well above the recent average, so
330/// the caller can drop it before the optimizer sees it.
331///
332/// The average is an exponential moving mean of the accepted losses. A loss
333/// that is not finite, or more than `ratio` times that mean once `warmup`
334/// losses have been accepted, is a spike: it is not folded into the mean, and
335/// [`SpikeGuard::consecutive`] counts how many came in a row. Skip a spike by
336/// clearing the gradients instead of stepping; after several in a row, roll
337/// back to the last checkpoint and its optimizer state.
338///
339/// ```
340/// use rusting_brain::SpikeGuard;
341///
342/// let mut guard = SpikeGuard::new(2.0, 0.99, 10);
343/// for loss in [3.0, 2.9, 2.8, 2.9, 2.7, 2.8, 2.6, 2.7, 2.6, 2.5] {
344///     assert!(!guard.is_spike(loss));
345/// }
346/// assert!(guard.is_spike(9.0));
347/// assert!(guard.is_spike(f32::NAN));
348/// assert_eq!(guard.consecutive(), 2);
349/// assert!(!guard.is_spike(2.6));
350/// assert_eq!(guard.consecutive(), 0);
351///
352/// // In a training loop:
353/// // let loss = model.accumulate_step(&batch)?;
354/// // if guard.is_spike(loss.lm_loss) {
355/// //     model.zero_grad();
356/// //     if guard.consecutive() >= 3 {
357/// //         model = TransformerLm::load_bin("last.bin")?;
358/// //         model.load_optimizer_state("last.opt")?;
359/// //     }
360/// // } else {
361/// //     model.step_clipped(1.0, 1.0)?;
362/// // }
363/// ```
364#[derive(Clone, Copy, Debug, PartialEq)]
365pub struct SpikeGuard {
366    ratio: f32,
367    beta: f32,
368    warmup: usize,
369    mean: f32,
370    accepted: usize,
371    consecutive: usize,
372}
373
374impl SpikeGuard {
375    /// A loss above `ratio` times the moving mean is a spike. `beta` is the
376    /// mean's decay per accepted loss (0.99 averages about the last hundred),
377    /// and no loss is a spike on size alone until `warmup` have been accepted.
378    pub fn new(ratio: f32, beta: f32, warmup: usize) -> Self {
379        Self {
380            ratio,
381            beta,
382            warmup,
383            mean: 0.0,
384            accepted: 0,
385            consecutive: 0,
386        }
387    }
388
389    /// Records `loss` and says whether it is a spike.
390    pub fn is_spike(&mut self, loss: f32) -> bool {
391        let spike =
392            !loss.is_finite() || (self.accepted >= self.warmup && loss > self.ratio * self.mean);
393        if spike {
394            self.consecutive += 1;
395            return true;
396        }
397        self.consecutive = 0;
398        self.accepted += 1;
399        // Bias-corrected, so the first losses are not pulled toward zero.
400        let weight = (1.0 - self.beta) / (1.0 - self.beta.powi(self.accepted as i32));
401        self.mean += weight * (loss - self.mean);
402        false
403    }
404
405    /// Spikes seen in a row since the last accepted loss.
406    pub fn consecutive(&self) -> usize {
407        self.consecutive
408    }
409
410    /// The moving mean of the accepted losses, 0 before the first.
411    pub fn mean(&self) -> f32 {
412        self.mean
413    }
414}
415
416#[cfg(test)]
417mod tests {
418    use super::*;
419
420    #[test]
421    fn a_spike_is_skipped_without_moving_the_mean() {
422        let mut guard = SpikeGuard::new(2.0, 0.9, 3);
423        // A large first loss is accepted: the warm-up has no mean to compare.
424        assert!(!guard.is_spike(10.0));
425        assert!((guard.mean() - 10.0).abs() < 1e-6);
426        assert!(!guard.is_spike(4.0));
427        assert!(!guard.is_spike(4.0));
428        let mean = guard.mean();
429        assert!(guard.is_spike(2.0 * mean + 0.1));
430        assert!(guard.is_spike(f32::INFINITY));
431        assert_eq!(guard.mean(), mean);
432        assert_eq!(guard.consecutive(), 2);
433        assert!(!guard.is_spike(2.0 * mean - 0.1));
434        assert_eq!(guard.consecutive(), 0);
435    }
436
437    #[test]
438    fn warmup_rises_to_the_peak_and_the_cosine_falls_to_the_floor() {
439        let schedule = Schedule::warmup_cosine(1.0, 10, 110);
440
441        assert!((schedule.rate(0) - 0.1).abs() < 1e-6);
442        assert!((schedule.rate(9) - 1.0).abs() < 1e-6);
443        // First step of the decay is still the peak: the cosine starts at one.
444        assert!((schedule.rate(10) - 1.0).abs() < 1e-6);
445        assert!((schedule.rate(60) - 0.55).abs() < 1e-3); // halfway, cosine 0.5
446        assert!((schedule.rate(110) - 0.1).abs() < 1e-6);
447        // Past the end, not below it.
448        assert!((schedule.rate(10_000) - 0.1).abs() < 1e-6);
449        // Monotonic across the decay.
450        for step in 10..110 {
451            assert!(schedule.rate(step) >= schedule.rate(step + 1));
452        }
453    }
454
455    #[test]
456    fn the_floor_is_adjustable_and_a_schedule_with_no_decay_is_flat() {
457        assert!(Schedule::warmup_cosine(1.0, 0, 100).floor(0.0).rate(100) < 1e-6);
458        // Warmup as long as the run: nothing left to decay over.
459        assert!((Schedule::warmup_cosine(1.0, 100, 100).rate(100) - 0.1).abs() < 1e-6);
460    }
461
462    #[test]
463    fn warmup_stable_decay_holds_the_peak_then_falls_linearly_to_the_floor() {
464        // Warmup 10, stable to step 80, linear decay over the last 20 steps.
465        let schedule = Schedule::warmup_stable_decay(1.0, 10, 100, 0.2).floor(0.0);
466
467        assert!((schedule.rate(0) - 0.1).abs() < 1e-6);
468        for step in 9..=80 {
469            assert_eq!(schedule.rate(step), 1.0);
470        }
471        assert!((schedule.rate(90) - 0.5).abs() < 1e-6);
472        assert!(schedule.rate(100) < 1e-6);
473        assert!(schedule.rate(10_000) < 1e-6);
474        // Extending the run before the decay leaves every rate so far alone.
475        let longer = Schedule::warmup_stable_decay(1.0, 10, 200, 0.2).floor(0.0);
476        for step in 0..=80 {
477            assert_eq!(schedule.rate(step), longer.rate(step));
478        }
479        // The default floor is a tenth, as for the cosine.
480        assert!((Schedule::warmup_stable_decay(1.0, 10, 100, 0.2).rate(100) - 0.1).abs() < 1e-6);
481    }
482
483    #[test]
484    fn setting_the_rate_keeps_the_rest_of_the_optimizer() {
485        let mut optimizer = Optimizer::adam_with_weight_decay(1e-3, 0.1);
486        optimizer.set_learning_rate(2e-4);
487
488        assert_eq!(optimizer.learning_rate(), 2e-4);
489        assert_eq!(optimizer, Optimizer::adam_with_weight_decay(2e-4, 0.1));
490    }
491}