use std::num::{NonZeroU64, NonZeroUsize};
use crate::error::GprError;
#[derive(Clone, Debug, PartialEq)]
pub struct Adam {
learning_rate: f64,
beta1: f64,
beta2: f64,
epsilon: f64,
batch_size: NonZeroUsize,
epochs: NonZeroU64,
seed: u64,
}
impl Default for Adam {
fn default() -> Self {
Self {
learning_rate: 1e-3,
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
batch_size: match NonZeroUsize::new(32) {
Some(n) => n,
None => NonZeroUsize::MIN,
},
epochs: match NonZeroU64::new(100) {
Some(n) => n,
None => NonZeroU64::MIN,
},
seed: 0,
}
}
}
impl Adam {
pub fn new() -> Self {
Self::default()
}
pub fn with_learning_rate(mut self, learning_rate: f64) -> Result<Self, GprError> {
if !learning_rate.is_finite() || learning_rate <= 0.0 {
return Err(GprError::InvalidConfig {
reason: "Adam learning rate must be finite and > 0".to_owned(),
});
}
self.learning_rate = learning_rate;
Ok(self)
}
pub fn with_beta1(mut self, beta1: f64) -> Result<Self, GprError> {
if !beta1.is_finite() || !(0.0..1.0).contains(&beta1) {
return Err(GprError::InvalidConfig {
reason: "Adam β1 must be finite and in [0, 1)".to_owned(),
});
}
self.beta1 = beta1;
Ok(self)
}
pub fn with_beta2(mut self, beta2: f64) -> Result<Self, GprError> {
if !beta2.is_finite() || !(0.0..1.0).contains(&beta2) {
return Err(GprError::InvalidConfig {
reason: "Adam β2 must be finite and in [0, 1)".to_owned(),
});
}
self.beta2 = beta2;
Ok(self)
}
pub fn with_epsilon(mut self, epsilon: f64) -> Result<Self, GprError> {
if !epsilon.is_finite() || epsilon <= 0.0 {
return Err(GprError::InvalidConfig {
reason: "Adam ε must be finite and > 0".to_owned(),
});
}
self.epsilon = epsilon;
Ok(self)
}
pub fn with_batch_size(mut self, batch_size: NonZeroUsize) -> Self {
self.batch_size = batch_size;
self
}
pub fn with_epochs(mut self, epochs: NonZeroU64) -> Self {
self.epochs = epochs;
self
}
pub fn with_seed(mut self, seed: u64) -> Self {
self.seed = seed;
self
}
pub(crate) fn batch_size(&self) -> usize {
self.batch_size.get()
}
pub(crate) fn epochs(&self) -> u64 {
self.epochs.get()
}
pub(crate) fn seed(&self) -> u64 {
self.seed
}
pub(crate) fn step(
&self,
z: &mut [f64],
grad: &[f64],
moment1: &mut [f64],
moment2: &mut [f64],
timestep: &mut u64,
) {
*timestep += 1;
let t = *timestep as f64;
let corr1 = 1.0 - self.beta1.powf(t);
let corr2 = 1.0 - self.beta2.powf(t);
for i in 0..z.len() {
moment1[i] = self.beta1 * moment1[i] + (1.0 - self.beta1) * grad[i];
moment2[i] = self.beta2 * moment2[i] + (1.0 - self.beta2) * grad[i] * grad[i];
let mhat = moment1[i] / corr1;
let vhat = moment2[i] / corr2;
z[i] -= self.learning_rate * mhat / (vhat.sqrt() + self.epsilon);
}
}
}