use tch::{Tensor, nn::VarStore};
use crate::{Result, RustTorchError};
mod schedulers;
mod stateful;
pub use schedulers::{
CosineAnnealingLr, ExponentialLr, MultiStepLr, PlateauMode, ReduceLrOnPlateau, StepLr,
ThresholdMode,
};
use stateful::Algorithm;
pub use stateful::{Optimizer, OptimizerState, ParameterGroup};
const DEFAULT_LEARNING_RATE: f64 = 1e-3;
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Adam {
learning_rate: f64,
beta1: f64,
beta2: f64,
eps: f64,
weight_decay: f64,
amsgrad: bool,
maximize: bool,
foreach: Option<bool>,
capturable: bool,
differentiable: bool,
fused: Option<bool>,
decoupled_weight_decay: bool,
}
impl Default for Adam {
fn default() -> Self {
Self {
learning_rate: DEFAULT_LEARNING_RATE,
beta1: 0.9,
beta2: 0.999,
eps: 1e-8,
weight_decay: 0.0,
amsgrad: false,
maximize: false,
foreach: None,
capturable: false,
differentiable: false,
fused: None,
decoupled_weight_decay: false,
}
}
}
impl Adam {
#[must_use]
pub fn builder() -> Self {
Self::default()
}
#[must_use]
pub const fn learning_rate(mut self, learning_rate: f64) -> Self {
self.learning_rate = learning_rate;
self
}
#[must_use]
pub const fn betas(mut self, beta1: f64, beta2: f64) -> Self {
self.beta1 = beta1;
self.beta2 = beta2;
self
}
#[must_use]
pub const fn beta1(mut self, beta1: f64) -> Self {
self.beta1 = beta1;
self
}
#[must_use]
pub const fn beta2(mut self, beta2: f64) -> Self {
self.beta2 = beta2;
self
}
#[must_use]
pub const fn eps(mut self, eps: f64) -> Self {
self.eps = eps;
self
}
#[must_use]
pub const fn weight_decay(mut self, weight_decay: f64) -> Self {
self.weight_decay = weight_decay;
self
}
#[must_use]
pub const fn amsgrad(mut self, amsgrad: bool) -> Self {
self.amsgrad = amsgrad;
self
}
#[must_use]
pub const fn maximize(mut self, maximize: bool) -> Self {
self.maximize = maximize;
self
}
#[must_use]
pub const fn foreach(mut self, foreach: bool) -> Self {
self.foreach = Some(foreach);
self
}
#[must_use]
pub const fn capturable(mut self, capturable: bool) -> Self {
self.capturable = capturable;
self
}
#[must_use]
pub const fn differentiable(mut self, differentiable: bool) -> Self {
self.differentiable = differentiable;
self
}
#[must_use]
pub const fn fused(mut self, fused: bool) -> Self {
self.fused = Some(fused);
self
}
#[must_use]
pub const fn decoupled_weight_decay(mut self, decoupled: bool) -> Self {
self.decoupled_weight_decay = decoupled;
self
}
pub fn build(self, var_store: &VarStore) -> Result<Optimizer> {
self.validate()?;
Optimizer::new(
var_store,
Algorithm::Adam {
beta1: self.beta1,
beta2: self.beta2,
eps: self.eps,
amsgrad: self.amsgrad,
decoupled: false,
},
self.learning_rate,
self.weight_decay,
)
}
fn validate(self) -> Result<()> {
validate_non_negative("learning_rate", self.learning_rate)?;
validate_beta("beta1", self.beta1)?;
validate_beta("beta2", self.beta2)?;
validate_non_negative("eps", self.eps)?;
validate_non_negative("weight_decay", self.weight_decay)?;
reject_unsupported("Adam", "maximize", self.maximize)?;
reject_unsupported("Adam", "foreach", self.foreach == Some(true))?;
reject_unsupported("Adam", "capturable", self.capturable)?;
reject_unsupported("Adam", "differentiable", self.differentiable)?;
reject_unsupported("Adam", "fused", self.fused == Some(true))?;
reject_unsupported(
"Adam",
"decoupled_weight_decay",
self.decoupled_weight_decay,
)
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Sgd {
learning_rate: f64,
momentum: f64,
dampening: f64,
weight_decay: f64,
nesterov: bool,
maximize: bool,
foreach: Option<bool>,
differentiable: bool,
fused: Option<bool>,
}
impl Default for Sgd {
fn default() -> Self {
Self {
learning_rate: DEFAULT_LEARNING_RATE,
momentum: 0.0,
dampening: 0.0,
weight_decay: 0.0,
nesterov: false,
maximize: false,
foreach: None,
differentiable: false,
fused: None,
}
}
}
impl Sgd {
#[must_use]
pub fn builder() -> Self {
Self::default()
}
#[must_use]
pub const fn learning_rate(mut self, learning_rate: f64) -> Self {
self.learning_rate = learning_rate;
self
}
#[must_use]
pub const fn momentum(mut self, momentum: f64) -> Self {
self.momentum = momentum;
self
}
#[must_use]
pub const fn dampening(mut self, dampening: f64) -> Self {
self.dampening = dampening;
self
}
#[must_use]
pub const fn weight_decay(mut self, weight_decay: f64) -> Self {
self.weight_decay = weight_decay;
self
}
#[must_use]
pub const fn nesterov(mut self, nesterov: bool) -> Self {
self.nesterov = nesterov;
self
}
#[must_use]
pub const fn maximize(mut self, maximize: bool) -> Self {
self.maximize = maximize;
self
}
#[must_use]
pub const fn foreach(mut self, foreach: bool) -> Self {
self.foreach = Some(foreach);
self
}
#[must_use]
pub const fn differentiable(mut self, differentiable: bool) -> Self {
self.differentiable = differentiable;
self
}
#[must_use]
pub const fn fused(mut self, fused: bool) -> Self {
self.fused = Some(fused);
self
}
pub fn build(self, var_store: &VarStore) -> Result<Optimizer> {
self.validate()?;
Optimizer::new(
var_store,
Algorithm::Sgd {
momentum: self.momentum,
dampening: self.dampening,
nesterov: self.nesterov,
},
self.learning_rate,
self.weight_decay,
)
}
fn validate(self) -> Result<()> {
validate_non_negative("learning_rate", self.learning_rate)?;
validate_non_negative("momentum", self.momentum)?;
validate_non_negative("dampening", self.dampening)?;
validate_non_negative("weight_decay", self.weight_decay)?;
if self.nesterov && (self.momentum <= 0.0 || self.dampening != 0.0) {
return Err(RustTorchError::InvalidConfiguration {
field: "nesterov",
reason: "requires momentum > 0 and dampening == 0".to_owned(),
});
}
reject_unsupported("SGD", "maximize", self.maximize)?;
reject_unsupported("SGD", "foreach", self.foreach == Some(true))?;
reject_unsupported("SGD", "differentiable", self.differentiable)?;
reject_unsupported("SGD", "fused", self.fused == Some(true))
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct AdamW {
config: Adam,
}
impl Default for AdamW {
fn default() -> Self {
Self {
config: Adam::default().weight_decay(0.01),
}
}
}
impl AdamW {
#[must_use]
pub fn builder() -> Self {
Self::default()
}
#[must_use]
pub const fn learning_rate(mut self, value: f64) -> Self {
self.config = self.config.learning_rate(value);
self
}
#[must_use]
pub const fn betas(mut self, beta1: f64, beta2: f64) -> Self {
self.config = self.config.betas(beta1, beta2);
self
}
#[must_use]
pub const fn eps(mut self, value: f64) -> Self {
self.config = self.config.eps(value);
self
}
#[must_use]
pub const fn weight_decay(mut self, value: f64) -> Self {
self.config = self.config.weight_decay(value);
self
}
#[must_use]
pub const fn amsgrad(mut self, enabled: bool) -> Self {
self.config = self.config.amsgrad(enabled);
self
}
pub fn build(self, var_store: &VarStore) -> Result<Optimizer> {
self.config.validate()?;
Optimizer::new(
var_store,
Algorithm::Adam {
beta1: self.config.beta1,
beta2: self.config.beta2,
eps: self.config.eps,
amsgrad: self.config.amsgrad,
decoupled: true,
},
self.config.learning_rate,
self.config.weight_decay,
)
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct RmsProp {
learning_rate: f64,
alpha: f64,
eps: f64,
weight_decay: f64,
momentum: f64,
centered: bool,
}
impl Default for RmsProp {
fn default() -> Self {
Self {
learning_rate: 0.01,
alpha: 0.99,
eps: 1e-8,
weight_decay: 0.0,
momentum: 0.0,
centered: false,
}
}
}
impl RmsProp {
#[must_use]
pub fn builder() -> Self {
Self::default()
}
#[must_use]
pub const fn learning_rate(mut self, value: f64) -> Self {
self.learning_rate = value;
self
}
#[must_use]
pub const fn alpha(mut self, value: f64) -> Self {
self.alpha = value;
self
}
#[must_use]
pub const fn eps(mut self, value: f64) -> Self {
self.eps = value;
self
}
#[must_use]
pub const fn weight_decay(mut self, value: f64) -> Self {
self.weight_decay = value;
self
}
#[must_use]
pub const fn momentum(mut self, value: f64) -> Self {
self.momentum = value;
self
}
#[must_use]
pub const fn centered(mut self, value: bool) -> Self {
self.centered = value;
self
}
pub fn build(self, var_store: &VarStore) -> Result<Optimizer> {
for (name, value) in [
("learning_rate", self.learning_rate),
("alpha", self.alpha),
("eps", self.eps),
("weight_decay", self.weight_decay),
("momentum", self.momentum),
] {
validate_non_negative(name, value)?;
}
Optimizer::new(
var_store,
Algorithm::RmsProp {
alpha: self.alpha,
eps: self.eps,
momentum: self.momentum,
centered: self.centered,
},
self.learning_rate,
self.weight_decay,
)
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Adagrad {
learning_rate: f64,
lr_decay: f64,
weight_decay: f64,
initial_accumulator: f64,
eps: f64,
}
impl Default for Adagrad {
fn default() -> Self {
Self {
learning_rate: 0.01,
lr_decay: 0.,
weight_decay: 0.,
initial_accumulator: 0.,
eps: 1e-10,
}
}
}
impl Adagrad {
#[must_use]
pub fn builder() -> Self {
Self::default()
}
#[must_use]
pub const fn learning_rate(mut self, value: f64) -> Self {
self.learning_rate = value;
self
}
#[must_use]
pub const fn lr_decay(mut self, value: f64) -> Self {
self.lr_decay = value;
self
}
#[must_use]
pub const fn weight_decay(mut self, value: f64) -> Self {
self.weight_decay = value;
self
}
#[must_use]
pub const fn initial_accumulator_value(mut self, value: f64) -> Self {
self.initial_accumulator = value;
self
}
#[must_use]
pub const fn eps(mut self, value: f64) -> Self {
self.eps = value;
self
}
pub fn build(self, store: &VarStore) -> Result<Optimizer> {
Optimizer::new(
store,
Algorithm::Adagrad {
lr_decay: self.lr_decay,
initial_accumulator: self.initial_accumulator,
eps: self.eps,
},
self.learning_rate,
self.weight_decay,
)
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Adadelta {
learning_rate: f64,
rho: f64,
eps: f64,
weight_decay: f64,
}
impl Default for Adadelta {
fn default() -> Self {
Self {
learning_rate: 1.,
rho: 0.9,
eps: 1e-6,
weight_decay: 0.,
}
}
}
impl Adadelta {
#[must_use]
pub fn builder() -> Self {
Self::default()
}
#[must_use]
pub const fn learning_rate(mut self, value: f64) -> Self {
self.learning_rate = value;
self
}
#[must_use]
pub const fn rho(mut self, value: f64) -> Self {
self.rho = value;
self
}
#[must_use]
pub const fn eps(mut self, value: f64) -> Self {
self.eps = value;
self
}
#[must_use]
pub const fn weight_decay(mut self, value: f64) -> Self {
self.weight_decay = value;
self
}
pub fn build(self, store: &VarStore) -> Result<Optimizer> {
Optimizer::new(
store,
Algorithm::Adadelta {
rho: self.rho,
eps: self.eps,
},
self.learning_rate,
self.weight_decay,
)
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Adamax {
learning_rate: f64,
beta1: f64,
beta2: f64,
eps: f64,
weight_decay: f64,
}
impl Default for Adamax {
fn default() -> Self {
Self {
learning_rate: 0.002,
beta1: 0.9,
beta2: 0.999,
eps: 1e-8,
weight_decay: 0.,
}
}
}
impl Adamax {
#[must_use]
pub fn builder() -> Self {
Self::default()
}
#[must_use]
pub const fn learning_rate(mut self, value: f64) -> Self {
self.learning_rate = value;
self
}
#[must_use]
pub const fn betas(mut self, beta1: f64, beta2: f64) -> Self {
self.beta1 = beta1;
self.beta2 = beta2;
self
}
#[must_use]
pub const fn eps(mut self, value: f64) -> Self {
self.eps = value;
self
}
#[must_use]
pub const fn weight_decay(mut self, value: f64) -> Self {
self.weight_decay = value;
self
}
pub fn build(self, store: &VarStore) -> Result<Optimizer> {
Optimizer::new(
store,
Algorithm::Adamax {
beta1: self.beta1,
beta2: self.beta2,
eps: self.eps,
},
self.learning_rate,
self.weight_decay,
)
}
}
fn validate_loss(loss: &Tensor) -> Result<()> {
if !loss.defined() {
return Err(RustTorchError::InvalidConfiguration {
field: "loss",
reason: "must be a defined scalar tensor".to_owned(),
});
}
let shape = loss.size();
if !shape.is_empty() {
return Err(RustTorchError::InvalidDimensions {
context: "optimizer loss".to_owned(),
expected: "scalar tensor with shape []".to_owned(),
actual: format!("shape {shape:?}"),
});
}
Ok(())
}
fn validate_non_negative(field: &'static str, value: f64) -> Result<()> {
if value.is_finite() && value >= 0.0 {
Ok(())
} else {
Err(RustTorchError::InvalidConfiguration {
field,
reason: format!("must be finite and non-negative, got {value}"),
})
}
}
fn validate_beta(field: &'static str, value: f64) -> Result<()> {
if value.is_finite() && (0.0..1.0).contains(&value) {
Ok(())
} else {
Err(RustTorchError::InvalidConfiguration {
field,
reason: format!("must be finite and in [0, 1), got {value}"),
})
}
}
fn reject_unsupported(component: &'static str, option: &'static str, enabled: bool) -> Result<()> {
if enabled {
Err(RustTorchError::UnsupportedOption { component, option })
} else {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use tch::{Device, Kind};
#[test]
fn rejects_invalid_and_unsupported_configuration() {
assert!(matches!(
Adam::builder().betas(1.0, 0.999).validate(),
Err(RustTorchError::InvalidConfiguration { field: "beta1", .. })
));
assert!(matches!(
Adam::builder().maximize(true).validate(),
Err(RustTorchError::UnsupportedOption {
option: "maximize",
..
})
));
assert!(matches!(
Sgd::builder().nesterov(true).validate(),
Err(RustTorchError::InvalidConfiguration {
field: "nesterov",
..
})
));
}
#[test]
fn requires_a_defined_scalar_loss() {
assert!(matches!(
validate_loss(&Tensor::new()),
Err(RustTorchError::InvalidConfiguration { field: "loss", .. })
));
assert!(matches!(
validate_loss(&Tensor::zeros([1], (Kind::Float, Device::Cpu))),
Err(RustTorchError::InvalidDimensions { .. })
));
assert!(validate_loss(&Tensor::zeros([], (Kind::Float, Device::Cpu))).is_ok());
}
}