use crate::utils::training::run_progress;
pub const DEFAULT_STEP_SIZE: usize = 10;
pub const DEFAULT_GAMMA: f32 = 0.5;
pub const DEFAULT_MIN_RATIO: f32 = 0.1;
pub const DEFAULT_MILESTONES: [usize; 2] = [50, 100];
pub const DEFAULT_WARMUP_FRAC: f32 = 0.3;
pub const DEFAULT_PERIOD: usize = 20;
pub const DEFAULT_WARMUP_STEPS: usize = 50;
#[derive(Clone, Debug, PartialEq)]
pub enum LrSchedule {
Constant,
StepDecay { step_size: usize, gamma: f32 },
Exponential { gamma: f32 },
MultiStep { gamma: f32, milestones: Vec<usize> },
Cosine { min_ratio: f32 },
OneCycle { warmup_frac: f32 },
Cyclic { period: usize, min_ratio: f32 },
Warmup { steps: usize },
}
impl LrSchedule {
pub const NAMES: &'static [&'static str] = &[
"constant",
"step_decay",
"exponential",
"multistep",
"cosine",
"onecycle",
"cyclic",
"warmup",
];
pub fn parse(name: &str) -> Option<Self> {
let name = name.trim();
if let Some((head, rest)) = name.split_once('(') {
let arg = rest.strip_suffix(')')?.trim();
let args: Vec<&str> = if arg.is_empty() {
Vec::new()
} else {
arg.split(',').map(str::trim).collect()
};
let num = |i: usize| -> Option<f32> { args.get(i)?.parse().ok() };
let count = |i: usize| -> Option<usize> { args.get(i)?.parse().ok() };
return match head.trim() {
"step_decay" => Some(Self::StepDecay {
step_size: count(0)?,
gamma: num(1)?,
}),
"exponential" => Some(Self::Exponential { gamma: num(0)? }),
"multistep" => Some(Self::MultiStep {
gamma: num(0)?,
milestones: args[1..].iter().filter_map(|m| m.parse().ok()).collect(),
}),
"cosine" => Some(Self::Cosine { min_ratio: num(0)? }),
"onecycle" => Some(Self::OneCycle {
warmup_frac: num(0)?,
}),
"cyclic" => Some(Self::Cyclic {
period: count(0)?,
min_ratio: num(1)?,
}),
"warmup" => Some(Self::Warmup { steps: count(0)? }),
"constant" => Some(Self::Constant),
_ => None,
};
}
match name {
"constant" | "none" | "off" => Some(Self::Constant),
"step_decay" | "step" => Some(Self::StepDecay {
step_size: DEFAULT_STEP_SIZE,
gamma: DEFAULT_GAMMA,
}),
"exponential" | "exp" => Some(Self::Exponential { gamma: 0.99 }),
"multistep" | "multi_step" => Some(Self::MultiStep {
gamma: DEFAULT_GAMMA,
milestones: DEFAULT_MILESTONES.to_vec(),
}),
"cosine" => Some(Self::Cosine {
min_ratio: DEFAULT_MIN_RATIO,
}),
"onecycle" | "one_cycle" => Some(Self::OneCycle {
warmup_frac: DEFAULT_WARMUP_FRAC,
}),
"cyclic" => Some(Self::Cyclic {
period: DEFAULT_PERIOD,
min_ratio: DEFAULT_MIN_RATIO,
}),
"warmup" => Some(Self::Warmup {
steps: DEFAULT_WARMUP_STEPS,
}),
_ => None,
}
}
pub fn label(&self) -> String {
match self {
Self::Constant => "constant".to_string(),
Self::StepDecay { step_size, gamma } => format!("step_decay({step_size},{gamma})"),
Self::Exponential { gamma } => format!("exponential({gamma})"),
Self::MultiStep { gamma, milestones } => {
let mut out = format!("multistep({gamma}");
for m in milestones {
out.push_str(&format!(",{m}"));
}
out.push(')');
out
}
Self::Cosine { min_ratio } => format!("cosine({min_ratio})"),
Self::OneCycle { warmup_frac } => format!("onecycle({warmup_frac})"),
Self::Cyclic { period, min_ratio } => format!("cyclic({period},{min_ratio})"),
Self::Warmup { steps } => format!("warmup({steps})"),
}
}
pub fn lr(&self, base: f32, step: usize, max_steps: Option<usize>) -> f32 {
let progress = || run_progress(step, max_steps);
match self {
Self::Constant => base,
Self::StepDecay { step_size, gamma } => {
let step_size = (*step_size).max(1);
base * gamma.powi((step / step_size) as i32)
}
Self::Exponential { gamma } => base * gamma.powi(step as i32),
Self::MultiStep { gamma, milestones } => {
let hits = milestones.iter().filter(|m| **m <= step).count() as i32;
base * gamma.powi(hits)
}
Self::Cosine { min_ratio } => match progress() {
Some(p) => {
let floor = base * min_ratio;
let shape = 0.5 * (1.0 + (std::f32::consts::PI * p).cos());
floor + (base - floor) * shape
}
None => base,
},
Self::OneCycle { warmup_frac } => match progress() {
Some(p) => {
let warm = warmup_frac.clamp(0.0, 1.0);
if p < warm {
let t = if warm > 0.0 { p / warm } else { 1.0 };
base * (0.1 + 0.9 * t)
} else {
let t = if warm < 1.0 {
(p - warm) / (1.0 - warm)
} else {
1.0
};
let floor = base * 0.01;
let shape = 0.5 * (1.0 + (std::f32::consts::PI * t).cos());
floor + (base - floor) * shape
}
}
None => base,
},
Self::Cyclic { period, min_ratio } => {
let period = (*period).max(1);
let t = (step % period) as f32 / period as f32;
let floor = base * min_ratio;
let shape = 0.5 * (1.0 - (2.0 * std::f32::consts::PI * t).cos());
floor + (base - floor) * shape
}
Self::Warmup { steps } => {
let steps = (*steps).max(1);
let t = ((step + 1) as f32 / steps as f32).min(1.0);
base * t
}
}
}
}
impl std::fmt::Display for LrSchedule {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.label())
}
}
impl std::str::FromStr for LrSchedule {
type Err = crate::utils::error::EngineError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Self::parse(s).ok_or_else(|| {
crate::utils::error::EngineError::InvalidOptions(format!(
"unknown lr_schedule {s:?} — expected one of {}",
Self::NAMES.join(", ")
))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn names_and_labels_round_trip() {
for name in LrSchedule::NAMES {
let s = LrSchedule::parse(name).unwrap_or_else(|| panic!("{name} should parse"));
assert_eq!(LrSchedule::parse(&s.label()), Some(s), "{name}");
}
for label in [
"step_decay(4,0.25)",
"exponential(0.9)",
"multistep(0.5,10,20,40)",
"cosine(0.05)",
"onecycle(0.2)",
"cyclic(7,0.2)",
"warmup(12)",
] {
let s = LrSchedule::parse(label).unwrap_or_else(|| panic!("{label} should parse"));
assert_eq!(s.label(), label, "label drifted");
}
assert!(LrSchedule::parse("cosine()").is_none());
assert!(LrSchedule::parse("cosine(nope)").is_none());
assert!(LrSchedule::parse("plateau").is_none());
}
#[test]
fn lr_is_a_pure_function_of_the_clock() {
let budget = Some(20usize);
for schedule in [
LrSchedule::StepDecay {
step_size: 5,
gamma: 0.5,
},
LrSchedule::Exponential { gamma: 0.9 },
LrSchedule::Cosine { min_ratio: 0.1 },
LrSchedule::OneCycle { warmup_frac: 0.3 },
LrSchedule::Cyclic {
period: 6,
min_ratio: 0.2,
},
LrSchedule::Warmup { steps: 8 },
] {
for step in [0usize, 1, 7, 19] {
let a = schedule.lr(1e-3, step, budget);
let b = schedule.lr(1e-3, step, budget);
assert_eq!(a, b, "{} at {step}", schedule.label());
assert!(
a.is_finite() && a >= 0.0,
"{} at {step} → {a}",
schedule.label()
);
}
}
}
#[test]
fn the_named_shapes_move_the_rate_the_way_their_names_say() {
let base = 1e-3f32;
let budget = Some(10usize);
assert_eq!(LrSchedule::Constant.lr(base, 0, budget), base);
assert_eq!(LrSchedule::Constant.lr(base, 9, budget), base);
let step = LrSchedule::StepDecay {
step_size: 5,
gamma: 0.5,
};
assert_eq!(step.lr(base, 4, budget), base);
assert_eq!(step.lr(base, 5, budget), base * 0.5);
let multi = LrSchedule::MultiStep {
gamma: 0.5,
milestones: vec![2, 6],
};
assert_eq!(multi.lr(base, 1, budget), base);
assert_eq!(multi.lr(base, 2, budget), base * 0.5);
assert_eq!(multi.lr(base, 6, budget), base * 0.25);
let cosine = LrSchedule::Cosine { min_ratio: 0.1 };
assert!((cosine.lr(base, 0, budget) - base).abs() < 1e-9);
assert!((cosine.lr(base, 10, budget) - base * 0.1).abs() < 1e-9);
let mut previous = f32::INFINITY;
for step in 0..=10 {
let value = cosine.lr(base, step, budget);
assert!(value <= previous + 1e-9, "cosine rose at {step}");
previous = value;
}
let one = LrSchedule::OneCycle { warmup_frac: 0.5 };
assert!(one.lr(base, 0, budget) < base);
assert!(one.lr(base, 4, budget) < base);
assert!((one.lr(base, 5, budget) - base).abs() < 1e-6);
assert!(one.lr(base, 10, budget) < base * 0.02);
let warm = LrSchedule::Warmup { steps: 4 };
assert!(warm.lr(base, 0, budget) < warm.lr(base, 1, budget));
assert_eq!(warm.lr(base, 3, budget), base);
assert_eq!(warm.lr(base, 9, budget), base);
let cyclic = LrSchedule::Cyclic {
period: 4,
min_ratio: 0.25,
};
assert!((cyclic.lr(base, 0, budget) - base * 0.25).abs() < 1e-6);
assert!((cyclic.lr(base, 2, budget) - base).abs() < 1e-6);
assert!((cyclic.lr(base, 4, budget) - base * 0.25).abs() < 1e-6);
}
#[test]
fn unbounded_runs_keep_the_base_rate() {
let base = 1e-3f32;
assert_eq!(
LrSchedule::Cosine { min_ratio: 0.1 }.lr(base, 7, None),
base
);
assert_eq!(
LrSchedule::OneCycle { warmup_frac: 0.3 }.lr(base, 7, None),
base
);
assert_eq!(
LrSchedule::Exponential { gamma: 0.5 }.lr(base, 2, None),
base * 0.25
);
}
}