rusting_brain/optimizers.rs
1//! Parameter update rules.
2//!
3//! The optimizer is a value, not a trait object, and it is re-read from the
4//! model on every step. Assigning a fresh one with a new learning rate is all
5//! a warmup or cosine schedule needs to do.
6
7use rayon::prelude::*;
8use serde::{Deserialize, Serialize};
9
10#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
11pub enum Optimizer {
12 Sgd {
13 learning_rate: f32,
14 },
15 Adam {
16 learning_rate: f32,
17 beta1: f32,
18 beta2: f32,
19 epsilon: f32,
20 /// Decoupled (AdamW-style) weight decay applied to weights only, not
21 /// biases. `0.0` reproduces the original unregularized Adam.
22 #[serde(default)]
23 weight_decay: f32,
24 },
25 /// Lion: the update is the *sign* of a momentum-smoothed gradient, so every
26 /// parameter moves by exactly the learning rate.
27 ///
28 /// It keeps one moment where Adam keeps two, and the sign makes the step
29 /// size independent of the gradient's magnitude, so the learning rate wants
30 /// to be three to ten times smaller than an Adam rate and the weight decay
31 /// correspondingly larger. `Param` still allocates both moment buffers,
32 /// because the optimizer is a value that can be swapped between steps, so
33 /// the memory Lion saves elsewhere is not saved here.
34 ///
35 /// The CPU and CUDA paths run it. Metal has no kernel for it and reports
36 /// that rather than quietly taking an Adam step.
37 Lion {
38 learning_rate: f32,
39 beta1: f32,
40 beta2: f32,
41 /// Decoupled weight decay, as on [`Optimizer::Adam`].
42 weight_decay: f32,
43 },
44}
45
46/// The sign of `value`, and zero for zero.
47///
48/// `f32::signum` gives `1.0` for a positive zero, which would push a parameter
49/// whose smoothed gradient has vanished.
50pub(crate) fn sign(value: f32) -> f32 {
51 if value > 0.0 {
52 1.0
53 } else if value < 0.0 {
54 -1.0
55 } else {
56 0.0
57 }
58}
59
60impl Optimizer {
61 pub fn sgd(learning_rate: f32) -> Self {
62 Self::Sgd { learning_rate }
63 }
64
65 pub fn adam(learning_rate: f32) -> Self {
66 Self::Adam {
67 learning_rate,
68 beta1: 0.9,
69 beta2: 0.999,
70 epsilon: 1e-8,
71 weight_decay: 0.0,
72 }
73 }
74
75 pub fn adam_with_weight_decay(learning_rate: f32, weight_decay: f32) -> Self {
76 Self::Adam {
77 learning_rate,
78 beta1: 0.9,
79 beta2: 0.999,
80 epsilon: 1e-8,
81 weight_decay,
82 }
83 }
84
85 /// Lion at `learning_rate`, with the betas from the paper.
86 ///
87 /// Start from roughly a tenth of the Adam rate that works for the same
88 /// model: the update is a sign, so the rate alone sets the step size.
89 pub fn lion(learning_rate: f32) -> Self {
90 Self::Lion {
91 learning_rate,
92 beta1: 0.9,
93 beta2: 0.99,
94 weight_decay: 0.0,
95 }
96 }
97
98 pub fn lion_with_weight_decay(learning_rate: f32, weight_decay: f32) -> Self {
99 Self::Lion {
100 learning_rate,
101 beta1: 0.9,
102 beta2: 0.99,
103 weight_decay,
104 }
105 }
106
107 /// Replaces the learning rate and keeps everything else.
108 ///
109 /// This is what a schedule should assign. Building a fresh
110 /// `Optimizer::adam(rate)` each step silently drops the betas, the epsilon
111 /// and the weight decay back to their defaults.
112 pub fn set_learning_rate(&mut self, rate: f32) {
113 match self {
114 Self::Sgd { learning_rate }
115 | Self::Adam { learning_rate, .. }
116 | Self::Lion { learning_rate, .. } => *learning_rate = rate,
117 }
118 }
119
120 /// The same optimizer with its weight decay set to zero.
121 pub(crate) fn without_weight_decay(&self) -> Self {
122 let mut optimizer = self.clone();
123 if let Self::Adam { weight_decay, .. } | Self::Lion { weight_decay, .. } = &mut optimizer {
124 *weight_decay = 0.0;
125 }
126 optimizer
127 }
128
129 /// The current learning rate, which is worth logging next to the loss.
130 pub fn learning_rate(&self) -> f32 {
131 match self {
132 Self::Sgd { learning_rate }
133 | Self::Adam { learning_rate, .. }
134 | Self::Lion { learning_rate, .. } => *learning_rate,
135 }
136 }
137}
138
139/// Linear warmup, then a cosine decay to a fraction of the peak.
140///
141/// Full-size steps before Adam's moment estimates have settled can knock a
142/// model somewhere it takes a long time to leave; full-size steps at the end
143/// stop it settling at all. Every transformer run wants this curve, and it is
144/// arithmetic rather than state, so it is a plain value the caller reads once
145/// per step:
146///
147/// ```
148/// # use rusting_brain::{Optimizer, Schedule};
149/// let schedule = Schedule::warmup_cosine(3e-4, 2_000, 100_000);
150/// let mut optimizer = Optimizer::adam(3e-4);
151/// for step in 0..10 {
152/// optimizer.set_learning_rate(schedule.rate(step));
153/// // model.optimizer = optimizer.clone();
154/// }
155/// ```
156#[derive(Clone, Copy, Debug, PartialEq)]
157pub struct Schedule {
158 peak: f32,
159 warmup: usize,
160 steps: usize,
161 floor: f32,
162 /// `Some(fraction)` for warmup-stable-decay; `None` for cosine.
163 linear_tail: Option<f32>,
164}
165
166impl Schedule {
167 /// `warmup` steps rising to `peak`, then a cosine decay over the rest of
168 /// `steps`, ending at a tenth of the peak.
169 pub fn warmup_cosine(peak: f32, warmup: usize, steps: usize) -> Self {
170 Self {
171 peak,
172 warmup,
173 steps,
174 floor: 0.1,
175 linear_tail: None,
176 }
177 }
178
179 /// Warmup-stable-decay: `warmup` steps rising to `peak`, the peak held,
180 /// then a linear decay to the floor over the last `decay_fraction` of
181 /// `steps`.
182 ///
183 /// Every rate before the decay starts is the peak whatever `steps` is, so
184 /// a run can be extended by raising `steps` before its decay begins.
185 pub fn warmup_stable_decay(
186 peak: f32,
187 warmup: usize,
188 steps: usize,
189 decay_fraction: f32,
190 ) -> Self {
191 Self {
192 linear_tail: Some(decay_fraction.clamp(0.0, 1.0)),
193 ..Self::warmup_cosine(peak, warmup, steps)
194 }
195 }
196
197 /// The fraction of `peak` the decay ends at. The default is `0.1`, and
198 /// `0.0` decays to nothing.
199 pub fn floor(mut self, fraction: f32) -> Self {
200 self.floor = fraction;
201 self
202 }
203
204 /// The rate for `step`, counted from zero.
205 ///
206 /// Steps past the end of the schedule hold at the floor rather than going
207 /// negative, so a run extended beyond its planned length keeps training.
208 pub fn rate(&self, step: usize) -> f32 {
209 if step < self.warmup {
210 // `step + 1`, so the first step is not a forward and backward pass
211 // at a learning rate of zero, which changes nothing.
212 return self.peak * (step + 1) as f32 / self.warmup as f32;
213 }
214 if let Some(fraction) = self.linear_tail {
215 let decay = (self.steps as f32 * fraction).round() as usize;
216 let start = self.steps.saturating_sub(decay).max(self.warmup);
217 if step < start {
218 return self.peak;
219 }
220 let progress = match self.steps > start {
221 true => ((step - start) as f32 / (self.steps - start) as f32).min(1.0),
222 false => 1.0,
223 };
224 return self.peak * (1.0 - (1.0 - self.floor) * progress);
225 }
226 let decay = self.steps.saturating_sub(self.warmup);
227 if decay == 0 {
228 return self.peak * self.floor;
229 }
230 let progress = ((step - self.warmup) as f32 / decay as f32).min(1.0);
231 let cosine = 0.5 * (1.0 + (std::f32::consts::PI * progress).cos());
232 self.peak * (self.floor + (1.0 - self.floor) * cosine)
233 }
234}
235
236/// The training-step trio: a global gradient norm, the optimizer step, and the
237/// clipped step that combines them.
238///
239/// Every model in the crate accumulates gradients into a flat list of
240/// [`Param`]s and then applies exactly this sequence. Two copies of gradient
241/// clipping that drift apart is the kind of bug that shows up as a flat loss
242/// curve six hours into a run and nowhere else, so there is one copy and the
243/// models delegate to it.
244///
245/// ```
246/// # use rusting_brain::optimizers::{Optimizer, step_clipped};
247/// # use rusting_brain::param::Param;
248/// # use rusting_brain::matrix::Matrix;
249/// let mut weight = Param::zeros(1, 2);
250/// weight.grad = Matrix::from_vec(1, 2, vec![3.0, 4.0]);
251/// let norm = step_clipped(&mut [&mut weight], &Optimizer::adam(1e-2), 1, 1.0, 1.0)?;
252/// assert!((norm - 5.0).abs() < 1e-6);
253/// # Ok::<(), rusting_brain::NetworkError>(())
254/// ```
255use crate::param::Param;
256
257/// The L2 norm of the accumulated gradients, over every parameter at once.
258///
259/// Worth logging on its own: a run that is about to diverge shows it in this
260/// number one or two steps before the loss moves.
261pub fn grad_norm(params: &mut [&mut Param]) -> Result<f32, crate::network::NetworkError> {
262 let mut total = 0.0;
263 for param in params.iter() {
264 total += param.grad_sum_squares()?;
265 }
266 Ok(total.sqrt() as f32)
267}
268
269/// Applies the optimizer to every parameter.
270///
271/// `scale` divides the accumulated gradient, so a caller that ran several
272/// batches before stepping passes `1.0 / batches`. `step` counts from 1, as
273/// in [`Param::step`](crate::param::Param::step), which panics on 0.
274pub fn apply_step(params: &mut [&mut Param], optimizer: &Optimizer, step: usize, scale: f32) {
275 if params.iter().any(|param| param.is_on_device()) {
276 // One stream, one cuBLAS handle: the launches would serialize anyway,
277 // and a rayon pool around them only adds contention.
278 for param in params.iter_mut() {
279 param.step(optimizer, step, scale);
280 }
281 return;
282 }
283 params
284 .par_iter_mut()
285 .for_each(|param| param.step(optimizer, step, scale));
286}
287
288/// [`apply_step`] with the gradients clipped to a global norm of `max_norm`
289/// first, and the pre-clip norm returned for logging.
290///
291/// Clipping is a uniform rescale of every gradient, and `scale` already
292/// multiplies every gradient uniformly, so this folds the clip into that factor
293/// rather than rewriting the gradient buffers.
294///
295/// A norm that is NaN or infinite skips the step: the gradients are cleared
296/// and the parameters and optimizer moments are left as they were, because
297/// one non-finite update would poison both for the rest of the run. The norm
298/// is still returned, so the caller sees the skip.
299pub fn step_clipped(
300 params: &mut [&mut Param],
301 optimizer: &Optimizer,
302 step: usize,
303 scale: f32,
304 max_norm: f32,
305) -> Result<f32, crate::network::NetworkError> {
306 let norm = grad_norm(params)?;
307 if !norm.is_finite() {
308 zero_grad(params);
309 return Ok(norm);
310 }
311 let clip = match norm > max_norm && norm > 0.0 {
312 true => max_norm / norm,
313 false => 1.0,
314 };
315 apply_step(params, optimizer, step, scale * clip);
316 Ok(norm)
317}
318
319pub fn zero_grad(params: &mut [&mut Param]) {
320 if params.iter().any(|param| param.is_on_device()) {
321 for param in params.iter_mut() {
322 param.zero_grad();
323 }
324 return;
325 }
326 params.par_iter_mut().for_each(|param| param.zero_grad());
327}
328
329/// Flags a training step whose loss jumps well above the recent average, so
330/// the caller can drop it before the optimizer sees it.
331///
332/// The average is an exponential moving mean of the accepted losses. A loss
333/// that is not finite, or more than `ratio` times that mean once `warmup`
334/// losses have been accepted, is a spike: it is not folded into the mean, and
335/// [`SpikeGuard::consecutive`] counts how many came in a row. Skip a spike by
336/// clearing the gradients instead of stepping; after several in a row, roll
337/// back to the last checkpoint and its optimizer state.
338///
339/// ```
340/// use rusting_brain::SpikeGuard;
341///
342/// let mut guard = SpikeGuard::new(2.0, 0.99, 10);
343/// for loss in [3.0, 2.9, 2.8, 2.9, 2.7, 2.8, 2.6, 2.7, 2.6, 2.5] {
344/// assert!(!guard.is_spike(loss));
345/// }
346/// assert!(guard.is_spike(9.0));
347/// assert!(guard.is_spike(f32::NAN));
348/// assert_eq!(guard.consecutive(), 2);
349/// assert!(!guard.is_spike(2.6));
350/// assert_eq!(guard.consecutive(), 0);
351///
352/// // In a training loop:
353/// // let loss = model.accumulate_step(&batch)?;
354/// // if guard.is_spike(loss.lm_loss) {
355/// // model.zero_grad();
356/// // if guard.consecutive() >= 3 {
357/// // model = TransformerLm::load_bin("last.bin")?;
358/// // model.load_optimizer_state("last.opt")?;
359/// // }
360/// // } else {
361/// // model.step_clipped(1.0, 1.0)?;
362/// // }
363/// ```
364#[derive(Clone, Copy, Debug, PartialEq)]
365pub struct SpikeGuard {
366 ratio: f32,
367 beta: f32,
368 warmup: usize,
369 mean: f32,
370 accepted: usize,
371 consecutive: usize,
372}
373
374impl SpikeGuard {
375 /// A loss above `ratio` times the moving mean is a spike. `beta` is the
376 /// mean's decay per accepted loss (0.99 averages about the last hundred),
377 /// and no loss is a spike on size alone until `warmup` have been accepted.
378 pub fn new(ratio: f32, beta: f32, warmup: usize) -> Self {
379 Self {
380 ratio,
381 beta,
382 warmup,
383 mean: 0.0,
384 accepted: 0,
385 consecutive: 0,
386 }
387 }
388
389 /// Records `loss` and says whether it is a spike.
390 pub fn is_spike(&mut self, loss: f32) -> bool {
391 let spike =
392 !loss.is_finite() || (self.accepted >= self.warmup && loss > self.ratio * self.mean);
393 if spike {
394 self.consecutive += 1;
395 return true;
396 }
397 self.consecutive = 0;
398 self.accepted += 1;
399 // Bias-corrected, so the first losses are not pulled toward zero.
400 let weight = (1.0 - self.beta) / (1.0 - self.beta.powi(self.accepted as i32));
401 self.mean += weight * (loss - self.mean);
402 false
403 }
404
405 /// Spikes seen in a row since the last accepted loss.
406 pub fn consecutive(&self) -> usize {
407 self.consecutive
408 }
409
410 /// The moving mean of the accepted losses, 0 before the first.
411 pub fn mean(&self) -> f32 {
412 self.mean
413 }
414}
415
416#[cfg(test)]
417mod tests {
418 use super::*;
419
420 #[test]
421 fn a_spike_is_skipped_without_moving_the_mean() {
422 let mut guard = SpikeGuard::new(2.0, 0.9, 3);
423 // A large first loss is accepted: the warm-up has no mean to compare.
424 assert!(!guard.is_spike(10.0));
425 assert!((guard.mean() - 10.0).abs() < 1e-6);
426 assert!(!guard.is_spike(4.0));
427 assert!(!guard.is_spike(4.0));
428 let mean = guard.mean();
429 assert!(guard.is_spike(2.0 * mean + 0.1));
430 assert!(guard.is_spike(f32::INFINITY));
431 assert_eq!(guard.mean(), mean);
432 assert_eq!(guard.consecutive(), 2);
433 assert!(!guard.is_spike(2.0 * mean - 0.1));
434 assert_eq!(guard.consecutive(), 0);
435 }
436
437 #[test]
438 fn warmup_rises_to_the_peak_and_the_cosine_falls_to_the_floor() {
439 let schedule = Schedule::warmup_cosine(1.0, 10, 110);
440
441 assert!((schedule.rate(0) - 0.1).abs() < 1e-6);
442 assert!((schedule.rate(9) - 1.0).abs() < 1e-6);
443 // First step of the decay is still the peak: the cosine starts at one.
444 assert!((schedule.rate(10) - 1.0).abs() < 1e-6);
445 assert!((schedule.rate(60) - 0.55).abs() < 1e-3); // halfway, cosine 0.5
446 assert!((schedule.rate(110) - 0.1).abs() < 1e-6);
447 // Past the end, not below it.
448 assert!((schedule.rate(10_000) - 0.1).abs() < 1e-6);
449 // Monotonic across the decay.
450 for step in 10..110 {
451 assert!(schedule.rate(step) >= schedule.rate(step + 1));
452 }
453 }
454
455 #[test]
456 fn the_floor_is_adjustable_and_a_schedule_with_no_decay_is_flat() {
457 assert!(Schedule::warmup_cosine(1.0, 0, 100).floor(0.0).rate(100) < 1e-6);
458 // Warmup as long as the run: nothing left to decay over.
459 assert!((Schedule::warmup_cosine(1.0, 100, 100).rate(100) - 0.1).abs() < 1e-6);
460 }
461
462 #[test]
463 fn warmup_stable_decay_holds_the_peak_then_falls_linearly_to_the_floor() {
464 // Warmup 10, stable to step 80, linear decay over the last 20 steps.
465 let schedule = Schedule::warmup_stable_decay(1.0, 10, 100, 0.2).floor(0.0);
466
467 assert!((schedule.rate(0) - 0.1).abs() < 1e-6);
468 for step in 9..=80 {
469 assert_eq!(schedule.rate(step), 1.0);
470 }
471 assert!((schedule.rate(90) - 0.5).abs() < 1e-6);
472 assert!(schedule.rate(100) < 1e-6);
473 assert!(schedule.rate(10_000) < 1e-6);
474 // Extending the run before the decay leaves every rate so far alone.
475 let longer = Schedule::warmup_stable_decay(1.0, 10, 200, 0.2).floor(0.0);
476 for step in 0..=80 {
477 assert_eq!(schedule.rate(step), longer.rate(step));
478 }
479 // The default floor is a tenth, as for the cosine.
480 assert!((Schedule::warmup_stable_decay(1.0, 10, 100, 0.2).rate(100) - 0.1).abs() < 1e-6);
481 }
482
483 #[test]
484 fn setting_the_rate_keeps_the_rest_of_the_optimizer() {
485 let mut optimizer = Optimizer::adam_with_weight_decay(1e-3, 0.1);
486 optimizer.set_learning_rate(2e-4);
487
488 assert_eq!(optimizer.learning_rate(), 2e-4);
489 assert_eq!(optimizer, Optimizer::adam_with_weight_decay(2e-4, 0.1));
490 }
491}