#[cfg(feature = "gpu")]
use crate::gpu::AdamWParams;
use crate::gpu::{GpuContext, GpuTensor};
use crate::Tensor;
use half::f16;
use serde::{Deserialize, Serialize};
use std::error::Error;
use std::fmt;
use std::fs;
use std::io::{BufReader, BufWriter};
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
fn default_true() -> bool {
true
}
fn default_one() -> f32 {
1.0
}
pub fn default_no_decay_embeddings() -> Vec<String> {
vec![
"tok_embeddings.weight".into(),
"lm_head.weight".into(),
]
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum OptimizerKind {
#[default]
AdamW,
Sgd,
Lion,
Adafactor,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum MomentDtype {
#[default]
Fp32,
Fp16,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AdamWConfig {
pub beta1: f32,
pub beta2: f32,
pub epsilon: f32,
pub weight_decay: f32,
#[serde(default = "default_true")]
pub no_decay_rms_norm: bool,
#[serde(default = "default_no_decay_embeddings")]
pub no_decay_embeddings: Vec<String>,
#[serde(default = "default_one")]
pub rms_norm_lr_scale: f32,
#[serde(default = "default_one")]
pub embedding_lr_scale: f32,
#[serde(default)]
pub kind: OptimizerKind,
#[serde(default)]
pub moment_dtype: MomentDtype,
#[serde(default)]
pub offload_moments: bool,
#[serde(default)]
pub momentum: f32,
}
impl Default for AdamWConfig {
fn default() -> Self {
Self {
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
weight_decay: 0.01,
no_decay_rms_norm: true,
no_decay_embeddings: default_no_decay_embeddings(),
rms_norm_lr_scale: 1.0,
embedding_lr_scale: 1.0,
kind: OptimizerKind::default(),
moment_dtype: MomentDtype::default(),
offload_moments: false,
momentum: 0.0,
}
}
}
impl AdamWConfig {
pub fn validate(&self) -> Result<(), OptimizerError> {
validate_config(self)
}
pub fn group_for(&self, name: &str) -> ParameterGroup {
let kind = parameter_kind(name);
let weight_decay = match kind {
ParameterKind::RmsNorm if self.no_decay_rms_norm => 0.0,
ParameterKind::Embedding
if self
.no_decay_embeddings
.iter()
.any(|candidate| candidate == name) =>
{
0.0
}
_ => self.weight_decay,
};
let lr_scale = match kind {
ParameterKind::RmsNorm => self.rms_norm_lr_scale,
ParameterKind::Embedding => self.embedding_lr_scale,
ParameterKind::Other => 1.0,
};
ParameterGroup {
weight_decay,
lr_scale,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
enum ParameterKind {
RmsNorm,
Embedding,
Other,
}
fn parameter_kind(name: &str) -> ParameterKind {
if name == "norm.weight"
|| name.ends_with(".attention_norm.weight")
|| name.ends_with(".ffn_norm.weight")
{
return ParameterKind::RmsNorm;
}
if name == "tok_embeddings.weight"
|| name == "lm_head.weight"
|| name == "lm_head.bias"
|| name.starts_with("tok_embeddings.")
|| name.starts_with("lm_head.")
{
return ParameterKind::Embedding;
}
ParameterKind::Other
}
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
pub struct ParameterGroup {
pub weight_decay: f32,
pub lr_scale: f32,
}
impl ParameterGroup {
pub fn uniform(weight_decay: f32) -> Self {
Self {
weight_decay,
lr_scale: 1.0,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum MomentStorage {
Fp32(Vec<f32>),
Fp16(Vec<u16>),
}
impl MomentStorage {
fn zeros(len: usize, dtype: MomentDtype) -> Self {
match dtype {
MomentDtype::Fp32 => MomentStorage::Fp32(vec![0.0; len]),
MomentDtype::Fp16 => MomentStorage::Fp16(vec![0u16; len]),
}
}
pub fn len(&self) -> usize {
match self {
MomentStorage::Fp32(values) => values.len(),
MomentStorage::Fp16(values) => values.len(),
}
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn dtype(&self) -> MomentDtype {
match self {
MomentStorage::Fp32(_) => MomentDtype::Fp32,
MomentStorage::Fp16(_) => MomentDtype::Fp16,
}
}
#[inline]
pub fn get(&self, index: usize) -> f32 {
match self {
MomentStorage::Fp32(values) => values[index],
MomentStorage::Fp16(values) => f16::from_bits(values[index]).to_f32(),
}
}
#[inline]
pub fn set(&mut self, index: usize, value: f32) {
match self {
MomentStorage::Fp32(values) => values[index] = value,
MomentStorage::Fp16(values) => values[index] = f16::from_f32(value).to_bits(),
}
}
pub fn to_f32_vec(&self) -> Vec<f32> {
match self {
MomentStorage::Fp32(values) => values.clone(),
MomentStorage::Fp16(values) => values
.iter()
.map(|&bits| f16::from_bits(bits).to_f32())
.collect(),
}
}
pub fn as_fp32(&self) -> Option<&[f32]> {
match self {
MomentStorage::Fp32(values) => Some(values),
MomentStorage::Fp16(_) => None,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ParameterMoments {
pub first: MomentStorage,
pub second: MomentStorage,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct AdamWState {
pub step: usize,
pub moments: Vec<ParameterMoments>,
}
#[derive(Debug)]
pub struct OptimizerError(pub String);
impl fmt::Display for OptimizerError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(formatter, "AdamW error: {}", self.0)
}
}
impl Error for OptimizerError {}
pub struct AdamW {
config: AdamWConfig,
state: AdamWState,
groups: Vec<ParameterGroup>,
gpu_moments: Option<Vec<(GpuTensor, GpuTensor)>>,
offload_path: Option<PathBuf>,
offload_id: u64,
moments_paged_out: bool,
}
fn next_offload_id() -> u64 {
static COUNTER: AtomicU64 = AtomicU64::new(0);
COUNTER.fetch_add(1, Ordering::Relaxed)
}
impl AdamW {
pub fn new(parameters: &[Tensor], config: AdamWConfig) -> Result<Self, OptimizerError> {
let groups = vec![ParameterGroup::uniform(config.weight_decay); parameters.len()];
Self::from_groups(parameters, groups, config)
}
pub fn new_grouped(
named_parameters: &[(String, Tensor)],
config: AdamWConfig,
) -> Result<Self, OptimizerError> {
let groups = named_parameters
.iter()
.map(|(name, _)| config.group_for(name))
.collect();
let parameters: Vec<Tensor> = named_parameters
.iter()
.map(|(_, parameter)| parameter.clone())
.collect();
Self::from_groups(¶meters, groups, config)
}
pub fn from_state(
parameters: &[Tensor],
config: AdamWConfig,
state: AdamWState,
) -> Result<Self, OptimizerError> {
let groups = vec![ParameterGroup::uniform(config.weight_decay); parameters.len()];
Self::from_groups_and_state(parameters, groups, config, state)
}
pub fn from_state_grouped(
named_parameters: &[(String, Tensor)],
config: AdamWConfig,
state: AdamWState,
) -> Result<Self, OptimizerError> {
let groups = named_parameters
.iter()
.map(|(name, _)| config.group_for(name))
.collect();
let parameters: Vec<Tensor> = named_parameters
.iter()
.map(|(_, parameter)| parameter.clone())
.collect();
Self::from_groups_and_state(¶meters, groups, config, state)
}
fn from_groups(
parameters: &[Tensor],
groups: Vec<ParameterGroup>,
config: AdamWConfig,
) -> Result<Self, OptimizerError> {
validate_config(&config)?;
validate_groups(parameters.len(), &groups)?;
let moments = parameters
.iter()
.map(|parameter| moments_for(parameter, &config))
.collect();
Ok(Self {
config,
state: AdamWState { step: 0, moments },
groups,
gpu_moments: None,
offload_path: None,
offload_id: next_offload_id(),
moments_paged_out: false,
})
}
fn from_groups_and_state(
parameters: &[Tensor],
groups: Vec<ParameterGroup>,
config: AdamWConfig,
state: AdamWState,
) -> Result<Self, OptimizerError> {
validate_config(&config)?;
validate_groups(parameters.len(), &groups)?;
validate_state(parameters, &config, &state)?;
Ok(Self {
config,
state,
groups,
gpu_moments: None,
offload_path: None,
offload_id: next_offload_id(),
moments_paged_out: false,
})
}
pub fn step(
&mut self,
parameters: &[Tensor],
learning_rate: f32,
) -> Result<(), OptimizerError> {
if !learning_rate.is_finite() || learning_rate < 0.0 {
return Err(OptimizerError(
"learning rate must be finite and non-negative".into(),
));
}
self.page_in_moments()?;
validate_state(parameters, &self.config, &self.state)?;
validate_groups(parameters.len(), &self.groups)?;
self.state.step += 1;
match self.config.kind {
OptimizerKind::AdamW => self.step_adamw(parameters, learning_rate),
OptimizerKind::Sgd => self.step_sgd(parameters, learning_rate),
OptimizerKind::Lion => self.step_lion(parameters, learning_rate),
OptimizerKind::Adafactor => self.step_adafactor(parameters, learning_rate),
}
if self.config.offload_moments {
self.page_out_moments()?;
}
Ok(())
}
fn step_adamw(&mut self, parameters: &[Tensor], learning_rate: f32) {
let step = self.state.step as f32;
let first_correction = 1.0 - self.config.beta1.powf(step);
let second_correction = 1.0 - self.config.beta2.powf(step);
let beta1 = self.config.beta1;
let beta2 = self.config.beta2;
let epsilon = self.config.epsilon;
for (index, (parameter, moments)) in
parameters.iter().zip(self.state.moments.iter_mut()).enumerate()
{
let Some(gradient) = parameter.grad() else {
continue;
};
let group = self.groups[index];
let effective_lr = learning_rate * group.lr_scale;
parameter.with_mut_data(|values| {
for i in 0..values.len() {
let first_moment = beta1 * moments.first.get(i) + (1.0 - beta1) * gradient[i];
let second_moment =
beta2 * moments.second.get(i) + (1.0 - beta2) * gradient[i] * gradient[i];
moments.first.set(i, first_moment);
moments.second.set(i, second_moment);
let first_hat = first_moment / first_correction;
let second_hat = second_moment / second_correction;
let adam_update = first_hat / (second_hat.sqrt() + epsilon);
values[i] -= effective_lr * (adam_update + group.weight_decay * values[i]);
}
});
}
}
fn step_sgd(&mut self, parameters: &[Tensor], learning_rate: f32) {
let momentum = self.config.momentum;
for (index, (parameter, moments)) in
parameters.iter().zip(self.state.moments.iter_mut()).enumerate()
{
let Some(gradient) = parameter.grad() else {
continue;
};
let group = self.groups[index];
let effective_lr = learning_rate * group.lr_scale;
parameter.with_mut_data(|values| {
for i in 0..values.len() {
let update = if momentum > 0.0 {
let velocity = momentum * moments.first.get(i) + gradient[i];
moments.first.set(i, velocity);
velocity
} else {
gradient[i]
};
values[i] -= effective_lr * (update + group.weight_decay * values[i]);
}
});
}
}
fn step_lion(&mut self, parameters: &[Tensor], learning_rate: f32) {
let beta1 = self.config.beta1;
let beta2 = self.config.beta2;
for (index, (parameter, moments)) in
parameters.iter().zip(self.state.moments.iter_mut()).enumerate()
{
let Some(gradient) = parameter.grad() else {
continue;
};
let group = self.groups[index];
let effective_lr = learning_rate * group.lr_scale;
parameter.with_mut_data(|values| {
for i in 0..values.len() {
let momentum = moments.first.get(i);
let update = (beta1 * momentum + (1.0 - beta1) * gradient[i]).signum();
let new_momentum = beta2 * momentum + (1.0 - beta2) * gradient[i];
moments.first.set(i, new_momentum);
values[i] -= effective_lr * (update + group.weight_decay * values[i]);
}
});
}
}
fn step_adafactor(&mut self, parameters: &[Tensor], learning_rate: f32) {
let beta2 = self.config.beta2;
let epsilon = self.config.epsilon;
for (index, (parameter, moments)) in
parameters.iter().zip(self.state.moments.iter_mut()).enumerate()
{
let Some(gradient) = parameter.grad() else {
continue;
};
let group = self.groups[index];
let effective_lr = learning_rate * group.lr_scale;
let shape = parameter.shape();
if shape.len() == 2 && shape[0] > 0 && shape[1] > 0 {
let rows = shape[0];
let cols = shape[1];
let mut row_means = vec![0.0f32; rows];
let mut col_means = vec![0.0f32; cols];
for r in 0..rows {
for c in 0..cols {
let g = gradient[r * cols + c];
let g2 = g * g;
row_means[r] += g2;
col_means[c] += g2;
}
}
for value in row_means.iter_mut() {
*value /= cols as f32;
}
for value in col_means.iter_mut() {
*value /= rows as f32;
}
for r in 0..rows {
let vr = beta2 * moments.first.get(r) + (1.0 - beta2) * row_means[r];
moments.first.set(r, vr);
}
for c in 0..cols {
let vc = beta2 * moments.second.get(c) + (1.0 - beta2) * col_means[c];
moments.second.set(c, vc);
}
let row_factor_mean: f32 =
(0..rows).map(|r| moments.first.get(r)).sum::<f32>() / rows as f32;
let normalizer = row_factor_mean.max(epsilon);
parameter.with_mut_data(|values| {
for r in 0..rows {
let vr = moments.first.get(r);
for c in 0..cols {
let vc = moments.second.get(c);
let flat = r * cols + c;
let v = (vr * vc / normalizer).max(0.0);
let update = gradient[flat] / (v.sqrt() + epsilon);
values[flat] -=
effective_lr * (update + group.weight_decay * values[flat]);
}
}
});
} else {
parameter.with_mut_data(|values| {
for i in 0..values.len() {
let g = gradient[i];
let second_moment = beta2 * moments.second.get(i) + (1.0 - beta2) * g * g;
moments.second.set(i, second_moment);
let update = g / (second_moment.sqrt() + epsilon);
values[i] -= effective_lr * (update + group.weight_decay * values[i]);
}
});
}
}
}
#[cfg(feature = "gpu")]
pub fn step_gpu(
&mut self,
parameters: &[Tensor],
learning_rate: f32,
gradient_scale: f32,
gpu: &GpuContext,
) -> Result<(), OptimizerError> {
if self.config.kind != OptimizerKind::AdamW {
return Err(OptimizerError(format!(
"GPU optimizer step only supports AdamW (got {:?}); SGD/Lion/Adafactor are CPU-only, use step() instead",
self.config.kind
)));
}
if self.config.moment_dtype != MomentDtype::Fp32 {
return Err(OptimizerError(
"GPU optimizer step only supports FP32 moments; FP16 moments are CPU-only, use step() instead"
.into(),
));
}
if !learning_rate.is_finite() || learning_rate < 0.0 {
return Err(OptimizerError(
"learning rate must be finite and non-negative".into(),
));
}
self.page_in_moments()?;
validate_state(parameters, &self.config, &self.state)?;
validate_groups(parameters.len(), &self.groups)?;
let needs_upload = self.gpu_moments.as_ref().is_none_or(|moments| {
moments
.first()
.is_some_and(|(first, _)| !first.belongs_to(gpu))
});
if needs_upload {
let mut uploaded = Vec::with_capacity(self.state.moments.len());
for moments in &self.state.moments {
let first_slice = moments.first.as_fp32().ok_or_else(|| {
OptimizerError("GPU optimizer step requires FP32 moment storage".into())
})?;
let second_slice = moments.second.as_fp32().ok_or_else(|| {
OptimizerError("GPU optimizer step requires FP32 moment storage".into())
})?;
let first = gpu
.upload(first_slice, vec![first_slice.len()])
.map_err(|error| OptimizerError(error.to_string()))?;
let second = gpu
.upload(second_slice, vec![second_slice.len()])
.map_err(|error| OptimizerError(error.to_string()))?;
uploaded.push((first, second));
}
self.gpu_moments = Some(uploaded);
}
self.state.step += 1;
let step = self.state.step as f32;
let first_correction = 1.0 - self.config.beta1.powf(step);
let second_correction = 1.0 - self.config.beta2.powf(step);
let gpu_moments = self.gpu_moments.as_ref().unwrap();
for (index, (parameter, (first, second))) in parameters.iter().zip(gpu_moments).enumerate()
{
let Some(gradient) = parameter.gpu_grad() else {
continue;
};
let group = self.groups[index];
let parameter_storage = parameter
.gpu_storage(gpu)
.map_err(|error| OptimizerError(error.to_string()))?;
gpu.adamw(
¶meter_storage,
&gradient,
first,
second,
AdamWParams {
length: u32::try_from(parameter.numel()).map_err(|_| {
OptimizerError(format!("parameter {index} exceeds GPU limits"))
})?,
padding_0: 0,
padding_1: 0,
padding_2: 0,
learning_rate: learning_rate * group.lr_scale,
beta1: self.config.beta1,
beta2: self.config.beta2,
epsilon: self.config.epsilon,
weight_decay: group.weight_decay,
first_correction,
second_correction,
gradient_scale,
},
)
.map_err(|error| OptimizerError(error.to_string()))?;
parameter.mark_gpu_authoritative();
}
Ok(())
}
#[cfg(not(feature = "gpu"))]
pub fn step_gpu(
&mut self,
_parameters: &[Tensor],
_learning_rate: f32,
_gradient_scale: f32,
_gpu: &GpuContext,
) -> Result<(), OptimizerError> {
Err(OptimizerError(
"GPU support is not compiled in; rebuild with `--features gpu`".into(),
))
}
pub fn page_out_moments(&mut self) -> Result<(), OptimizerError> {
if self.moments_paged_out {
return Ok(());
}
let path = self.ensure_offload_path()?;
let file = fs::File::create(&path).map_err(|error| {
OptimizerError(format!(
"failed to create moment offload file {}: {error}",
path.display()
))
})?;
serde_json::to_writer(BufWriter::new(file), &self.state.moments).map_err(|error| {
OptimizerError(format!("failed to write moments to offload file: {error}"))
})?;
self.state.moments = Vec::new();
self.moments_paged_out = true;
Ok(())
}
pub fn page_in_moments(&mut self) -> Result<(), OptimizerError> {
if !self.moments_paged_out {
return Ok(());
}
let path = self.offload_path.clone().ok_or_else(|| {
OptimizerError("moments are marked paged out but no offload path was recorded".into())
})?;
let file = fs::File::open(&path).map_err(|error| {
OptimizerError(format!(
"failed to open moment offload file {}: {error}",
path.display()
))
})?;
let moments: Vec<ParameterMoments> = serde_json::from_reader(BufReader::new(file))
.map_err(|error| {
OptimizerError(format!("failed to read moment offload file: {error}"))
})?;
self.state.moments = moments;
self.moments_paged_out = false;
Ok(())
}
fn ensure_offload_path(&mut self) -> Result<PathBuf, OptimizerError> {
if let Some(path) = &self.offload_path {
return Ok(path.clone());
}
let directory =
std::env::temp_dir().join(format!("oxidelm-moments-{}", std::process::id()));
fs::create_dir_all(&directory).map_err(|error| {
OptimizerError(format!(
"failed to create moment offload directory {}: {error}",
directory.display()
))
})?;
let path = directory.join(format!("optimizer-{}.json", self.offload_id));
self.offload_path = Some(path.clone());
Ok(path)
}
pub fn checkpoint_state(&self) -> Result<AdamWState, OptimizerError> {
if let Some(gpu_moments) = &self.gpu_moments {
let moments = gpu_moments
.iter()
.map(|(first, second)| {
Ok(ParameterMoments {
first: MomentStorage::Fp32(
first
.to_vec()
.map_err(|error| OptimizerError(error.to_string()))?,
),
second: MomentStorage::Fp32(
second
.to_vec()
.map_err(|error| OptimizerError(error.to_string()))?,
),
})
})
.collect::<Result<Vec<_>, OptimizerError>>()?;
return Ok(AdamWState {
step: self.state.step,
moments,
});
}
if self.moments_paged_out {
let path = self.offload_path.as_ref().ok_or_else(|| {
OptimizerError(
"moments are marked paged out but no offload path was recorded".into(),
)
})?;
let file = fs::File::open(path).map_err(|error| {
OptimizerError(format!(
"failed to open moment offload file {}: {error}",
path.display()
))
})?;
let moments: Vec<ParameterMoments> = serde_json::from_reader(BufReader::new(file))
.map_err(|error| {
OptimizerError(format!("failed to read moment offload file: {error}"))
})?;
return Ok(AdamWState {
step: self.state.step,
moments,
});
}
Ok(self.state.clone())
}
pub fn config(&self) -> &AdamWConfig {
&self.config
}
pub fn state(&self) -> &AdamWState {
&self.state
}
pub fn groups(&self) -> &[ParameterGroup] {
&self.groups
}
}
impl Drop for AdamW {
fn drop(&mut self) {
if let Some(path) = self.offload_path.take() {
let _ = fs::remove_file(path);
}
}
}
fn moment_layout(shape: &[usize], numel: usize, config: &AdamWConfig) -> (usize, usize) {
match config.kind {
OptimizerKind::AdamW => (numel, numel),
OptimizerKind::Sgd => {
if config.momentum > 0.0 {
(numel, 0)
} else {
(0, 0)
}
}
OptimizerKind::Lion => (numel, 0),
OptimizerKind::Adafactor => {
if shape.len() == 2 {
(shape[0], shape[1])
} else {
(0, numel)
}
}
}
}
fn moments_for(parameter: &Tensor, config: &AdamWConfig) -> ParameterMoments {
let (first_len, second_len) = moment_layout(¶meter.shape(), parameter.numel(), config);
let dtype = match config.kind {
OptimizerKind::AdamW | OptimizerKind::Lion => config.moment_dtype,
OptimizerKind::Sgd | OptimizerKind::Adafactor => MomentDtype::Fp32,
};
ParameterMoments {
first: MomentStorage::zeros(first_len, dtype),
second: MomentStorage::zeros(second_len, dtype),
}
}
fn validate_config(config: &AdamWConfig) -> Result<(), OptimizerError> {
if !(0.0..1.0).contains(&config.beta1) || !(0.0..1.0).contains(&config.beta2) {
return Err(OptimizerError("betas must be in [0, 1)".into()));
}
if config.epsilon <= 0.0 || config.weight_decay < 0.0 {
return Err(OptimizerError(
"epsilon must be positive and weight decay non-negative".into(),
));
}
if !config.rms_norm_lr_scale.is_finite()
|| config.rms_norm_lr_scale < 0.0
|| !config.embedding_lr_scale.is_finite()
|| config.embedding_lr_scale < 0.0
{
return Err(OptimizerError(
"parameter-group learning-rate scales must be finite and non-negative".into(),
));
}
if !config.momentum.is_finite() || !(0.0..1.0).contains(&config.momentum) {
return Err(OptimizerError(
"momentum must be finite and in [0, 1)".into(),
));
}
if config.moment_dtype == MomentDtype::Fp16
&& !matches!(config.kind, OptimizerKind::AdamW | OptimizerKind::Lion)
{
return Err(OptimizerError(
"FP16 moment storage is only supported for AdamW and Lion; Adafactor always uses \
factored FP32 moments and SGD state is FP32-only"
.into(),
));
}
Ok(())
}
fn validate_groups(parameter_count: usize, groups: &[ParameterGroup]) -> Result<(), OptimizerError> {
if groups.len() != parameter_count {
return Err(OptimizerError(
"parameter count does not match optimizer groups".into(),
));
}
for (index, group) in groups.iter().enumerate() {
if !group.weight_decay.is_finite()
|| group.weight_decay < 0.0
|| !group.lr_scale.is_finite()
|| group.lr_scale < 0.0
{
return Err(OptimizerError(format!(
"optimizer group {index} has invalid weight decay or learning-rate scale"
)));
}
}
Ok(())
}
fn validate_state(
parameters: &[Tensor],
config: &AdamWConfig,
state: &AdamWState,
) -> Result<(), OptimizerError> {
if parameters.len() != state.moments.len() {
return Err(OptimizerError(
"parameter count does not match optimizer state".into(),
));
}
for (index, (parameter, moments)) in parameters.iter().zip(state.moments.iter()).enumerate() {
let (expected_first, expected_second) =
moment_layout(¶meter.shape(), parameter.numel(), config);
if moments.first.len() != expected_first || moments.second.len() != expected_second {
return Err(OptimizerError(format!(
"optimizer moment length mismatch for parameter {index}"
)));
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{
default_no_decay_embeddings, AdamW, AdamWConfig, MomentDtype, OptimizerKind,
ParameterGroup,
};
use crate::Tensor;
#[test]
fn adamw_reduces_simple_quadratic_parameter() {
let parameter = Tensor::scalar(2.0, true);
let mut optimizer = AdamW::new(
&[parameter.clone()],
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
)
.unwrap();
parameter.powf(2.0).backward();
optimizer.step(&[parameter.clone()], 0.1).unwrap();
assert!(parameter.item() < 2.0);
assert!((parameter.item() - 1.9).abs() < 1e-4);
}
#[test]
fn grouped_config_excludes_norms_and_selected_embeddings_from_decay() {
let config = AdamWConfig {
weight_decay: 0.1,
no_decay_rms_norm: true,
no_decay_embeddings: vec!["tok_embeddings.weight".into()],
rms_norm_lr_scale: 0.5,
embedding_lr_scale: 0.25,
..Default::default()
};
assert_eq!(
config.group_for("layers.0.attention.q_proj.weight"),
ParameterGroup {
weight_decay: 0.1,
lr_scale: 1.0
}
);
assert_eq!(
config.group_for("layers.0.attention_norm.weight"),
ParameterGroup {
weight_decay: 0.0,
lr_scale: 0.5
}
);
assert_eq!(
config.group_for("norm.weight"),
ParameterGroup {
weight_decay: 0.0,
lr_scale: 0.5
}
);
assert_eq!(
config.group_for("tok_embeddings.weight"),
ParameterGroup {
weight_decay: 0.0,
lr_scale: 0.25
}
);
assert_eq!(
config.group_for("lm_head.weight"),
ParameterGroup {
weight_decay: 0.1,
lr_scale: 0.25
}
);
}
#[test]
fn new_grouped_applies_zero_weight_decay_to_embeddings() {
let embedding = Tensor::new(vec![1.0, -1.0], vec![2], true);
let projection = Tensor::new(vec![1.0, -1.0], vec![2], true);
let named = vec![
("tok_embeddings.weight".into(), embedding.clone()),
("layers.0.attention.q_proj.weight".into(), projection.clone()),
];
let mut optimizer = AdamW::new_grouped(
&named,
AdamWConfig {
weight_decay: 0.5,
no_decay_embeddings: default_no_decay_embeddings(),
..Default::default()
},
)
.unwrap();
let zero = Tensor::scalar(0.0, false);
embedding
.mul(&zero)
.add(&projection.mul(&zero))
.sum()
.backward();
let parameters = vec![embedding.clone(), projection.clone()];
optimizer.step(¶meters, 1.0).unwrap();
assert_eq!(embedding.data(), vec![1.0, -1.0]);
assert_eq!(projection.data(), vec![0.5, -0.5]);
}
#[test]
fn new_grouped_scales_learning_rate_per_group() {
let embedding = Tensor::scalar(1.0, true);
let projection = Tensor::scalar(1.0, true);
let named = vec![
("tok_embeddings.weight".into(), embedding.clone()),
("layers.0.attention.q_proj.weight".into(), projection.clone()),
];
let mut optimizer = AdamW::new_grouped(
&named,
AdamWConfig {
weight_decay: 0.0,
embedding_lr_scale: 0.5,
..Default::default()
},
)
.unwrap();
embedding.add(&projection).backward();
let parameters = vec![embedding.clone(), projection.clone()];
optimizer.step(¶meters, 0.1).unwrap();
let embedding_delta = (1.0 - embedding.item()).abs();
let projection_delta = (1.0 - projection.item()).abs();
assert!((embedding_delta * 2.0 - projection_delta).abs() < 1e-5);
}
#[test]
fn parameter_kind_classifies_norm_and_embedding_names() {
assert!(matches!(
super::parameter_kind("layers.3.ffn_norm.weight"),
super::ParameterKind::RmsNorm
));
assert!(matches!(
super::parameter_kind("tok_embeddings.weight"),
super::ParameterKind::Embedding
));
assert!(matches!(
super::parameter_kind("layers.0.attention.q_proj.weight"),
super::ParameterKind::Other
));
}
#[test]
fn fp16_adamw_moments_closely_track_fp32() {
let initial = vec![2.0f32, -3.0, 0.5, 1.25];
let fp32_param = Tensor::new(initial.clone(), vec![4], true);
let fp16_param = Tensor::new(initial, vec![4], true);
let mut fp32_optimizer = AdamW::new(
&[fp32_param.clone()],
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
)
.unwrap();
let mut fp16_optimizer = AdamW::new(
&[fp16_param.clone()],
AdamWConfig {
weight_decay: 0.0,
moment_dtype: MomentDtype::Fp16,
..Default::default()
},
)
.unwrap();
for _ in 0..8 {
fp32_param.zero_grad();
fp32_param.powf(2.0).sum().backward();
fp32_optimizer.step(&[fp32_param.clone()], 0.05).unwrap();
fp16_param.zero_grad();
fp16_param.powf(2.0).sum().backward();
fp16_optimizer.step(&[fp16_param.clone()], 0.05).unwrap();
}
for (fp32_value, fp16_value) in fp32_param.data().iter().zip(fp16_param.data().iter()) {
let diff = (fp32_value - fp16_value).abs();
let tolerance = 1e-2 * fp32_value.abs().max(1.0);
assert!(
diff < tolerance,
"fp32={fp32_value} fp16={fp16_value} diff={diff} tolerance={tolerance}"
);
}
}
#[test]
fn sgd_reduces_simple_quadratic() {
let parameter = Tensor::scalar(4.0, true);
let mut optimizer = AdamW::new(
&[parameter.clone()],
AdamWConfig {
kind: OptimizerKind::Sgd,
weight_decay: 0.0,
momentum: 0.9,
..Default::default()
},
)
.unwrap();
for _ in 0..20 {
parameter.zero_grad();
parameter.powf(2.0).backward();
optimizer.step(&[parameter.clone()], 0.05).unwrap();
}
assert!(parameter.item().abs() < 4.0);
assert!(parameter.item().is_finite());
}
#[test]
fn lion_reduces_simple_quadratic() {
let parameter = Tensor::scalar(3.0, true);
let mut optimizer = AdamW::new(
&[parameter.clone()],
AdamWConfig {
kind: OptimizerKind::Lion,
weight_decay: 0.0,
..Default::default()
},
)
.unwrap();
for _ in 0..20 {
parameter.zero_grad();
parameter.powf(2.0).backward();
optimizer.step(&[parameter.clone()], 0.05).unwrap();
}
assert!(parameter.item().abs() < 3.0);
assert!(parameter.item().is_finite());
}
#[test]
fn adafactor_reduces_2d_quadratic() {
let values = vec![2.0f32, -1.5, 0.8, -0.3, 1.2, -2.1];
let parameter = Tensor::new(values, vec![2, 3], true);
let mut optimizer = AdamW::new(
&[parameter.clone()],
AdamWConfig {
kind: OptimizerKind::Adafactor,
weight_decay: 0.0,
..Default::default()
},
)
.unwrap();
let initial_loss: f32 = parameter.data().iter().map(|v| v * v).sum();
for _ in 0..30 {
parameter.zero_grad();
parameter.powf(2.0).sum().backward();
optimizer.step(&[parameter.clone()], 0.1).unwrap();
}
let final_loss: f32 = parameter.data().iter().map(|v| v * v).sum();
assert!(
final_loss < initial_loss,
"expected loss to decrease: initial={initial_loss} final={final_loss}"
);
assert!(parameter.data().iter().all(|value| value.is_finite()));
}
#[test]
fn offload_moments_round_trips_through_disk_between_steps() {
let parameter = Tensor::scalar(2.0, true);
let mut optimizer = AdamW::new(
&[parameter.clone()],
AdamWConfig {
weight_decay: 0.0,
offload_moments: true,
..Default::default()
},
)
.unwrap();
parameter.powf(2.0).backward();
optimizer.step(&[parameter.clone()], 0.1).unwrap();
let after_first_step = parameter.item();
assert!(after_first_step < 2.0);
assert!(optimizer.moments_paged_out);
let offload_path = optimizer
.offload_path
.clone()
.expect("offload path should be recorded once paged out");
assert!(
offload_path.exists(),
"moment offload file should exist between steps"
);
parameter.zero_grad();
parameter.powf(2.0).backward();
optimizer.step(&[parameter.clone()], 0.1).unwrap();
assert!(parameter.item() < after_first_step);
assert!(optimizer.moments_paged_out);
}
#[test]
fn validate_rejects_fp16_moments_outside_adamw_and_lion() {
let mut config = AdamWConfig {
kind: OptimizerKind::Adafactor,
moment_dtype: MomentDtype::Fp16,
..Default::default()
};
assert!(config.validate().is_err());
config.kind = OptimizerKind::Sgd;
assert!(config.validate().is_err());
config.kind = OptimizerKind::Lion;
assert!(config.validate().is_ok());
config.kind = OptimizerKind::AdamW;
assert!(config.validate().is_ok());
}
#[test]
fn moments_are_sized_according_to_optimizer_kind() {
let parameter = Tensor::new(vec![0.0; 6], vec![2, 3], true);
let sgd_no_momentum = AdamW::new(
&[parameter.clone()],
AdamWConfig {
kind: OptimizerKind::Sgd,
momentum: 0.0,
..Default::default()
},
)
.unwrap();
assert_eq!(sgd_no_momentum.state().moments[0].first.len(), 0);
assert_eq!(sgd_no_momentum.state().moments[0].second.len(), 0);
let sgd_with_momentum = AdamW::new(
&[parameter.clone()],
AdamWConfig {
kind: OptimizerKind::Sgd,
momentum: 0.9,
..Default::default()
},
)
.unwrap();
assert_eq!(sgd_with_momentum.state().moments[0].first.len(), 6);
assert_eq!(sgd_with_momentum.state().moments[0].second.len(), 0);
let lion = AdamW::new(
&[parameter.clone()],
AdamWConfig {
kind: OptimizerKind::Lion,
..Default::default()
},
)
.unwrap();
assert_eq!(lion.state().moments[0].first.len(), 6);
assert_eq!(lion.state().moments[0].second.len(), 0);
let adafactor = AdamW::new(
&[parameter.clone()],
AdamWConfig {
kind: OptimizerKind::Adafactor,
..Default::default()
},
)
.unwrap();
assert_eq!(adafactor.state().moments[0].first.len(), 2);
assert_eq!(adafactor.state().moments[0].second.len(), 3);
let adamw = AdamW::new(&[parameter], AdamWConfig::default()).unwrap();
assert_eq!(adamw.state().moments[0].first.len(), 6);
assert_eq!(adamw.state().moments[0].second.len(), 6);
}
}