use crate::graph::network::Network;
use flodl::nn::Module;
use flodl::nn::optim::{Adagrad, Adam, AdamW, NAdam, Optimizer, RAdam, RMSprop, SGD, Stateful};
use std::path::Path;
pub const DEFAULT_ADAMW_WEIGHT_DECAY: f64 = 0.01;
pub const DEFAULT_SGD_MOMENTUM: f64 = 0.9;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum OptimizerKind {
#[default]
Adam,
AdamW,
Sgd,
Rmsprop,
Adagrad,
NAdam,
RAdam,
}
impl OptimizerKind {
pub const NAMES: &'static [&'static str] = &[
"adam", "adamw", "sgd", "rmsprop", "adagrad", "nadam", "radam",
];
pub fn parse(name: &str) -> Option<Self> {
match name.trim().to_lowercase().as_str() {
"adam" => Some(Self::Adam),
"adamw" | "adam_w" => Some(Self::AdamW),
"sgd" | "momentum" => Some(Self::Sgd),
"rmsprop" | "rms" => Some(Self::Rmsprop),
"adagrad" => Some(Self::Adagrad),
"nadam" => Some(Self::NAdam),
"radam" => Some(Self::RAdam),
_ => None,
}
}
pub fn label(self) -> &'static str {
match self {
Self::Adam => "adam",
Self::AdamW => "adamw",
Self::Sgd => "sgd",
Self::Rmsprop => "rmsprop",
Self::Adagrad => "adagrad",
Self::NAdam => "nadam",
Self::RAdam => "radam",
}
}
pub fn make(self, net: &Network, lr: f32) -> Box<dyn Optimizer> {
let lr = lr as f64;
let params = net.parameters();
match self {
Self::Adam => Box::new(Adam::new(¶ms, lr)),
Self::AdamW => Box::new(AdamW::new(¶ms, lr, DEFAULT_ADAMW_WEIGHT_DECAY)),
Self::Sgd => Box::new(SGD::new(¶ms, lr, DEFAULT_SGD_MOMENTUM)),
Self::Rmsprop => Box::new(RMSprop::new(¶ms, lr)),
Self::Adagrad => Box::new(Adagrad::new(¶ms, lr)),
Self::NAdam => Box::new(NAdam::new(¶ms, lr)),
Self::RAdam => Box::new(RAdam::new(¶ms, lr)),
}
}
pub fn load(self, net: &Network, lr: f32, path: &Path) -> Option<Box<dyn Optimizer>> {
let path = path.to_string_lossy();
let lr = lr as f64;
let params = net.parameters();
match self {
Self::Adam => load_into(Adam::new(¶ms, lr), &path),
Self::AdamW => load_into(AdamW::new(¶ms, lr, DEFAULT_ADAMW_WEIGHT_DECAY), &path),
Self::Sgd => load_into(SGD::new(¶ms, lr, DEFAULT_SGD_MOMENTUM), &path),
Self::Rmsprop => load_into(RMSprop::new(¶ms, lr), &path),
Self::Adagrad => load_into(Adagrad::new(¶ms, lr), &path),
Self::NAdam => load_into(NAdam::new(¶ms, lr), &path),
Self::RAdam => load_into(RAdam::new(¶ms, lr), &path),
}
}
}
fn load_into<O: Optimizer + Stateful + 'static>(
mut opt: O,
path: &str,
) -> Option<Box<dyn Optimizer>> {
match opt.load_state_file(path) {
Ok(()) => Some(Box::new(opt)),
Err(e) => {
tracing::warn!("optimizer state {path} could not be loaded ({e}) — restarting moments");
None
}
}
}
impl std::fmt::Display for OptimizerKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.label())
}
}
impl std::str::FromStr for OptimizerKind {
type Err = crate::utils::error::EngineError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Self::parse(s).ok_or_else(|| {
crate::utils::error::EngineError::InvalidOptions(format!(
"unknown optimizer {s:?} — expected one of {}",
Self::NAMES.join(", ")
))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_name_parses_and_labels_round_trip() {
for name in OptimizerKind::NAMES {
let kind = OptimizerKind::parse(name).unwrap_or_else(|| panic!("{name} should parse"));
assert_eq!(kind.label(), *name);
assert_eq!(OptimizerKind::parse(kind.label()), Some(kind));
}
assert_eq!(OptimizerKind::parse("AdamW"), Some(OptimizerKind::AdamW));
assert_eq!(OptimizerKind::parse(" adam "), Some(OptimizerKind::Adam));
assert_eq!(OptimizerKind::parse("lbfgs"), None);
assert!("nope".parse::<OptimizerKind>().is_err());
assert_eq!(OptimizerKind::default().label(), "adam");
}
}