use super::{Optimizer, stateful::invalid, validate_non_negative};
use crate::Result;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct ScheduleState {
schema_version: u32,
epoch: u64,
group_ids: Vec<usize>,
base_rates: Vec<f64>,
last_rates: Vec<f64>,
}
impl ScheduleState {
fn new(optimizer: &Optimizer) -> Result<Self> {
let groups = optimizer.parameter_groups();
if groups.is_empty() {
return Err(invalid("a scheduler needs at least one parameter group"));
}
let rates: Vec<_> = groups.iter().map(|g| g.learning_rate).collect();
Ok(Self {
schema_version: 1,
epoch: 0,
group_ids: groups.iter().map(|g| g.id).collect(),
base_rates: rates.clone(),
last_rates: rates,
})
}
fn current(&self, optimizer: &Optimizer) -> Result<Vec<f64>> {
if self.schema_version != 1
|| self.epoch == u64::MAX
|| self.group_ids.is_empty()
|| self.group_ids.len() != self.base_rates.len()
|| self.group_ids.len() != self.last_rates.len()
|| self.group_ids.windows(2).any(|pair| pair[0] >= pair[1])
{
return Err(invalid("invalid scheduler state or epoch counter"));
}
for &rate in self.base_rates.iter().chain(&self.last_rates) {
validate_non_negative("learning_rate", rate)?;
}
let groups = optimizer.parameter_groups();
if groups.iter().map(|g| g.id).collect::<Vec<_>>() != self.group_ids {
return Err(invalid("scheduler parameter group identities changed"));
}
Ok(groups.into_iter().map(|g| g.learning_rate).collect())
}
fn apply(&mut self, optimizer: &mut Optimizer, rates: Vec<f64>, epoch: u64) -> Result<()> {
let rates_by_group = self
.group_ids
.iter()
.copied()
.zip(rates.iter().copied())
.collect::<Vec<_>>();
optimizer.apply_group_rates(&rates_by_group)?;
self.last_rates = rates;
self.epoch = epoch;
Ok(())
}
}
macro_rules! accessors {
() => {
pub const fn epoch(&self) -> u64 {
self.state.epoch
}
pub fn last_learning_rates(&self) -> &[f64] {
&self.state.last_rates
}
};
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StepLr {
state: ScheduleState,
step_size: u64,
gamma: f64,
}
impl StepLr {
pub fn new(optimizer: &Optimizer, step_size: u64, gamma: f64) -> Result<Self> {
positive(step_size, "step_size")?;
validate_non_negative("gamma", gamma)?;
Ok(Self {
state: ScheduleState::new(optimizer)?,
step_size,
gamma,
})
}
pub fn step(&mut self, optimizer: &mut Optimizer) -> Result<()> {
positive(self.step_size, "step_size")?;
validate_non_negative("gamma", self.gamma)?;
let mut rates = self.state.current(optimizer)?;
let epoch = self.state.epoch + 1;
if epoch.is_multiple_of(self.step_size) {
for rate in &mut rates {
*rate *= self.gamma;
}
}
self.state.apply(optimizer, rates, epoch)
}
accessors!();
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ExponentialLr {
state: ScheduleState,
gamma: f64,
}
impl ExponentialLr {
pub fn new(optimizer: &Optimizer, gamma: f64) -> Result<Self> {
validate_non_negative("gamma", gamma)?;
Ok(Self {
state: ScheduleState::new(optimizer)?,
gamma,
})
}
pub fn step(&mut self, optimizer: &mut Optimizer) -> Result<()> {
validate_non_negative("gamma", self.gamma)?;
let rates = self
.state
.current(optimizer)?
.into_iter()
.map(|rate| rate * self.gamma)
.collect();
self.state.apply(optimizer, rates, self.state.epoch + 1)
}
accessors!();
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MultiStepLr {
state: ScheduleState,
milestones: Vec<u64>,
gamma: f64,
}
impl MultiStepLr {
pub fn new(optimizer: &mut Optimizer, mut milestones: Vec<u64>, gamma: f64) -> Result<Self> {
validate_non_negative("gamma", gamma)?;
milestones.sort_unstable();
let mut state = ScheduleState::new(optimizer)?;
let factor = gamma.powf(milestones.iter().filter(|&&m| m == 0).count() as f64);
state.apply(
optimizer,
state.last_rates.iter().map(|rate| rate * factor).collect(),
0,
)?;
Ok(Self {
state,
milestones,
gamma,
})
}
pub fn step(&mut self, optimizer: &mut Optimizer) -> Result<()> {
validate_non_negative("gamma", self.gamma)?;
if self.milestones.windows(2).any(|p| p[0] > p[1]) {
return Err(invalid("scheduler milestones must be sorted"));
}
let rates = self.state.current(optimizer)?;
let epoch = self.state.epoch + 1;
let count = self
.milestones
.iter()
.filter(|&&milestone| milestone == epoch)
.count();
let rates = rates
.into_iter()
.map(|rate| rate * self.gamma.powf(count as f64))
.collect();
self.state.apply(optimizer, rates, epoch)
}
accessors!();
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CosineAnnealingLr {
state: ScheduleState,
t_max: u64,
eta_min: f64,
}
impl CosineAnnealingLr {
pub fn new(optimizer: &Optimizer, t_max: u64, eta_min: f64) -> Result<Self> {
positive(t_max, "t_max")?;
validate_non_negative("eta_min", eta_min)?;
if t_max > u64::MAX / 2 {
return Err(invalid("t_max is too large"));
}
Ok(Self {
state: ScheduleState::new(optimizer)?,
t_max,
eta_min,
})
}
pub fn step(&mut self, optimizer: &mut Optimizer) -> Result<()> {
positive(self.t_max, "t_max")?;
validate_non_negative("eta_min", self.eta_min)?;
if self.t_max > u64::MAX / 2 {
return Err(invalid("t_max is too large"));
}
let current = self.state.current(optimizer)?;
let epoch = self.state.epoch + 1;
let pi = std::f64::consts::PI;
let at_turn = epoch > self.t_max && (epoch - 1 - self.t_max).is_multiple_of(2 * self.t_max);
let rates = current
.iter()
.zip(&self.state.base_rates)
.map(|(&rate, &base)| {
if at_turn {
rate + (base - self.eta_min) * (1. - (pi / self.t_max as f64).cos()) / 2.
} else {
(1. + (pi * epoch as f64 / self.t_max as f64).cos())
/ (1. + (pi * (epoch - 1) as f64 / self.t_max as f64).cos())
* (rate - self.eta_min)
+ self.eta_min
}
})
.collect();
self.state.apply(optimizer, rates, epoch)
}
accessors!();
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum PlateauMode {
Min,
Max,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ThresholdMode {
Relative,
Absolute,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ReduceLrOnPlateau {
state: ScheduleState,
mode: PlateauMode,
factor: f64,
patience: u64,
threshold: f64,
threshold_mode: ThresholdMode,
cooldown: u64,
cooldown_remaining: u64,
min_rates: Vec<f64>,
eps: f64,
best: Option<f64>,
bad_epochs: u64,
}
impl ReduceLrOnPlateau {
pub fn new(optimizer: &Optimizer) -> Result<Self> {
let state = ScheduleState::new(optimizer)?;
let min_rates = vec![0.; state.group_ids.len()];
Ok(Self {
state,
mode: PlateauMode::Min,
factor: 0.1,
patience: 10,
threshold: 1e-4,
threshold_mode: ThresholdMode::Relative,
cooldown: 0,
cooldown_remaining: 0,
min_rates,
eps: 1e-8,
best: None,
bad_epochs: 0,
})
}
#[must_use]
pub fn mode(mut self, mode: PlateauMode) -> Self {
self.mode = mode;
self
}
pub fn factor(mut self, factor: f64) -> Result<Self> {
if !factor.is_finite() || !(0. ..1.).contains(&factor) {
return Err(invalid("plateau factor must be finite and in [0, 1)"));
}
self.factor = factor;
Ok(self)
}
#[must_use]
pub fn patience(mut self, patience: u64) -> Self {
self.patience = patience;
self
}
pub fn threshold(mut self, threshold: f64, mode: ThresholdMode) -> Result<Self> {
validate_non_negative("threshold", threshold)?;
self.threshold = threshold;
self.threshold_mode = mode;
Ok(self)
}
#[must_use]
pub fn cooldown(mut self, cooldown: u64) -> Self {
self.cooldown = cooldown;
self
}
pub fn min_learning_rates(mut self, rates: Vec<f64>) -> Result<Self> {
if rates.len() != self.state.group_ids.len() {
return Err(invalid("minimum rate count differs from group count"));
}
for &rate in &rates {
validate_non_negative("min_learning_rate", rate)?;
}
self.min_rates = rates;
Ok(self)
}
pub fn eps(mut self, eps: f64) -> Result<Self> {
validate_non_negative("eps", eps)?;
self.eps = eps;
Ok(self)
}
pub fn step(&mut self, optimizer: &mut Optimizer, metric: f64) -> Result<()> {
let mut rates = self.state.current(optimizer)?;
if !metric.is_finite()
|| !self.factor.is_finite()
|| !(0. ..1.).contains(&self.factor)
|| self.best.is_some_and(|best| !best.is_finite())
|| self.bad_epochs > self.state.epoch
|| self.cooldown_remaining > self.cooldown
|| self.min_rates.len() != rates.len()
{
return Err(invalid("invalid plateau metric or scheduler state"));
}
validate_non_negative("threshold", self.threshold)?;
validate_non_negative("eps", self.eps)?;
for &rate in &self.min_rates {
validate_non_negative("min_learning_rate", rate)?;
}
let improved = self
.best
.is_none_or(|best| match (self.mode, self.threshold_mode) {
(PlateauMode::Min, ThresholdMode::Relative) => {
metric < best * (1. - self.threshold)
}
(PlateauMode::Max, ThresholdMode::Relative) => {
metric > best * (1. + self.threshold)
}
(PlateauMode::Min, ThresholdMode::Absolute) => metric < best - self.threshold,
(PlateauMode::Max, ThresholdMode::Absolute) => metric > best + self.threshold,
});
let best = if improved { Some(metric) } else { self.best };
let mut bad = if improved { 0 } else { self.bad_epochs + 1 };
let mut cooldown = self.cooldown_remaining;
if cooldown > 0 {
cooldown -= 1;
bad = 0;
}
if bad > self.patience {
for (rate, &minimum) in rates.iter_mut().zip(&self.min_rates) {
let next = (*rate * self.factor).max(minimum);
if *rate - next > self.eps {
*rate = next;
}
}
cooldown = self.cooldown;
bad = 0;
}
self.state.apply(optimizer, rates, self.state.epoch + 1)?;
self.best = best;
self.bad_epochs = bad;
self.cooldown_remaining = cooldown;
Ok(())
}
accessors!();
}
fn positive(value: u64, field: &'static str) -> Result<()> {
if value == 0 {
Err(crate::RustTorchError::InvalidConfiguration {
field,
reason: "must be positive".into(),
})
} else {
Ok(())
}
}