use burn::module::AutodiffModule;
use burn::optim::adaptor::OptimizerAdaptor;
use burn::optim::{
Adam, AdamConfig, AdamW, AdamWConfig, GradientsParams, Optimizer, Sgd, SgdConfig,
};
use burn::tensor::backend::AutodiffBackend;
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
pub enum OptimiserConfig {
Adam {
lr: f64,
config: AdamConfig,
},
AdamW {
lr: f64,
config: AdamWConfig,
},
Sgd {
lr: f64,
config: SgdConfig,
},
}
impl Default for OptimiserConfig {
fn default() -> Self {
Self::AdamW {
lr: 3e-4,
config: AdamWConfig::new(),
}
}
}
impl OptimiserConfig {
pub fn lr(&self) -> f64 {
match self {
Self::Adam { lr, .. } | Self::AdamW { lr, .. } | Self::Sgd { lr, .. } => *lr,
}
}
pub fn init<B: AutodiffBackend, M: AutodiffModule<B>>(&self) -> Optimiser<B, M> {
let algorithm = match self {
Self::Adam { config, .. } => Algorithm::Adam(config.init()),
Self::AdamW { config, .. } => Algorithm::AdamW(config.init()),
Self::Sgd { config, .. } => Algorithm::Sgd(config.init()),
};
Optimiser {
lr: self.lr(),
algorithm,
}
}
}
pub struct Optimiser<B: AutodiffBackend, M: AutodiffModule<B>> {
lr: f64,
algorithm: Algorithm<B, M>,
}
enum Algorithm<B: AutodiffBackend, M: AutodiffModule<B>> {
Adam(OptimizerAdaptor<Adam, M, B>),
AdamW(OptimizerAdaptor<AdamW, M, B>),
Sgd(OptimizerAdaptor<Sgd<B::InnerBackend>, M, B>),
}
impl<B: AutodiffBackend, M: AutodiffModule<B>> Optimiser<B, M> {
pub fn step(&mut self, module: M, grads: GradientsParams) -> M {
match &mut self.algorithm {
Algorithm::Adam(optim) => optim.step(self.lr, module, grads),
Algorithm::AdamW(optim) => optim.step(self.lr, module, grads),
Algorithm::Sgd(optim) => optim.step(self.lr, module, grads),
}
}
}
#[cfg(test)]
mod tests {
use burn::backend::Autodiff;
use burn::backend::ndarray::{NdArray, NdArrayDevice};
use burn::grad_clipping::GradientClippingConfig;
use burn::nn::{Linear, LinearConfig};
use burn::optim::momentum::MomentumConfig;
use burn::tensor::Tensor;
use super::*;
type Ad = Autodiff<NdArray>;
fn loss(model: &Linear<Ad>, device: &NdArrayDevice) -> Tensor<Ad, 1> {
let input = Tensor::<Ad, 2>::from_floats([[1.0, 2.0], [-1.0, 0.5]], device);
let out = model.forward(input);
(out.clone() * out).sum().reshape([1])
}
fn value(loss: Tensor<Ad, 1>) -> f32 {
loss.into_data().to_vec::<f32>().unwrap()[0]
}
#[test]
fn every_optimiser_lowers_the_loss() {
let device = NdArrayDevice::default();
let configs = [
OptimiserConfig::default(),
OptimiserConfig::AdamW {
lr: 1e-2,
config: AdamWConfig::new(),
},
OptimiserConfig::Sgd {
lr: 1e-2,
config: SgdConfig::new()
.with_momentum(Some(MomentumConfig::new()))
.with_gradient_clipping(Some(GradientClippingConfig::Norm(10.0))),
},
];
for config in configs {
let config = match config {
OptimiserConfig::Adam { config, .. } => OptimiserConfig::Adam { lr: 1e-2, config },
other => other,
};
let mut model = LinearConfig::new(2, 2).init::<Ad>(&device);
let mut optimiser = config.init::<Ad, Linear<Ad>>();
let before = value(loss(&model, &device));
for _ in 0..20 {
let grads = GradientsParams::from_grads(loss(&model, &device).backward(), &model);
model = optimiser.step(model, grads);
}
let after = value(loss(&model, &device));
assert!(
after < before,
"{config:?}: loss went from {before} to {after}"
);
}
}
#[test]
fn the_configuration_survives_a_round_trip_through_json() {
let config = OptimiserConfig::Sgd {
lr: 0.5,
config: SgdConfig::new().with_momentum(Some(MomentumConfig::new())),
};
let json = serde_json::to_string(&config).unwrap();
let back: OptimiserConfig = serde_json::from_str(&json).unwrap();
assert_eq!(serde_json::to_string(&back).unwrap(), json);
assert_eq!(back.lr(), 0.5);
}
}