1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
//! Parameter update rules.
//!
//! The optimizer is a value, not a trait object, and it is re-read from the
//! model on every step. Assigning a fresh one with a new learning rate is all
//! a warmup or cosine schedule needs to do.
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub enum Optimizer {
Sgd {
learning_rate: f32,
},
Adam {
learning_rate: f32,
beta1: f32,
beta2: f32,
epsilon: f32,
/// Decoupled (AdamW-style) weight decay applied to weights only, not
/// biases. `0.0` reproduces the original unregularized Adam.
#[serde(default)]
weight_decay: f32,
},
}
impl Optimizer {
pub fn sgd(learning_rate: f32) -> Self {
Self::Sgd { learning_rate }
}
pub fn adam(learning_rate: f32) -> Self {
Self::Adam {
learning_rate,
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
weight_decay: 0.0,
}
}
pub fn adam_with_weight_decay(learning_rate: f32, weight_decay: f32) -> Self {
Self::Adam {
learning_rate,
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
weight_decay,
}
}
}