use std::{
collections::{BTreeMap, BTreeSet},
sync::{
Arc, Mutex,
atomic::{AtomicU64, Ordering},
},
};
use serde::{Deserialize, Serialize};
use tch::{
COptimizer, Device, Kind, Tensor,
nn::{VarStore, Variables},
};
use super::{validate_beta, validate_loss, validate_non_negative};
use crate::{Result, RustTorchError, no_grad};
static NEXT_OPTIMIZER_ID: AtomicU64 = AtomicU64::new(1);
const SCHEMA: u32 = 1;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) enum Algorithm {
Adam {
beta1: f64,
beta2: f64,
eps: f64,
amsgrad: bool,
decoupled: bool,
},
Sgd {
momentum: f64,
dampening: f64,
nesterov: bool,
},
RmsProp {
alpha: f64,
eps: f64,
momentum: f64,
centered: bool,
},
Adagrad {
lr_decay: f64,
initial_accumulator: f64,
eps: f64,
},
Adadelta {
rho: f64,
eps: f64,
},
Adamax {
beta1: f64,
beta2: f64,
eps: f64,
},
}
impl Algorithm {
fn validate(&self) -> Result<()> {
match *self {
Self::Adam {
beta1, beta2, eps, ..
}
| Self::Adamax { beta1, beta2, eps } => {
validate_beta("beta1", beta1)?;
validate_beta("beta2", beta2)?;
validate_non_negative("eps", eps)
}
Self::Sgd {
momentum,
dampening,
nesterov,
} => {
validate_non_negative("momentum", momentum)?;
validate_non_negative("dampening", dampening)?;
if nesterov && (momentum <= 0. || dampening != 0.) {
return Err(invalid(
"nesterov requires positive momentum and zero dampening",
));
}
Ok(())
}
Self::RmsProp {
alpha,
eps,
momentum,
..
} => {
validate_non_negative("alpha", alpha)?;
validate_non_negative("eps", eps)?;
validate_non_negative("momentum", momentum)
}
Self::Adagrad {
lr_decay,
initial_accumulator,
eps,
} => {
validate_non_negative("lr_decay", lr_decay)?;
validate_non_negative("initial_accumulator", initial_accumulator)?;
validate_non_negative("eps", eps)
}
Self::Adadelta { rho, eps } => {
if !rho.is_finite() || !(0. ..=1.).contains(&rho) {
return Err(invalid("rho must be finite and in [0, 1]"));
}
validate_non_negative("eps", eps)
}
}
}
fn slots(&self) -> &'static [&'static str] {
match self {
Self::Adam { amsgrad: true, .. } => &["exp_avg", "exp_avg_sq", "max_exp_avg_sq"],
Self::Adam { .. } => &["exp_avg", "exp_avg_sq"],
Self::Sgd { momentum, .. } if *momentum != 0. => &["momentum_buffer"],
Self::Sgd { .. } => &[],
Self::RmsProp {
centered: true,
momentum,
..
} if *momentum != 0. => &["square_avg", "grad_avg", "momentum_buffer"],
Self::RmsProp { centered: true, .. } => &["square_avg", "grad_avg"],
Self::RmsProp { momentum, .. } if *momentum != 0. => &["square_avg", "momentum_buffer"],
Self::RmsProp { .. } => &["square_avg"],
Self::Adagrad { .. } => &["sum"],
Self::Adadelta { .. } => &["square_avg", "acc_delta"],
Self::Adamax { .. } => &["exp_avg", "exp_inf"],
}
}
fn same_family(&self, other: &Self) -> bool {
std::mem::discriminant(self) == std::mem::discriminant(other)
&& match (self, other) {
(Self::Adam { decoupled: a, .. }, Self::Adam { decoupled: b, .. }) => a == b,
_ => true,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ParameterGroup {
pub id: usize,
pub learning_rate: f64,
pub weight_decay: f64,
pub parameters: Vec<String>,
}
#[derive(Debug)]
struct Parameter {
tensor: Tensor,
group: usize,
step: u64,
slots: BTreeMap<String, Tensor>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OptimizerState {
schema_version: u32,
algorithm: Algorithm,
default_learning_rate: f64,
default_weight_decay: f64,
completed_steps: u64,
groups: Vec<ParameterGroup>,
parameters: BTreeMap<String, ParameterState>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct ParameterState {
shape: Vec<i64>,
dtype: Dtype,
group: usize,
step: u64,
slots: BTreeMap<String, TensorState>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
enum Dtype {
Float32,
Float64,
Float16,
BFloat16,
}
impl Dtype {
fn from_kind(kind: Kind) -> Result<Self> {
match kind {
Kind::Float => Ok(Self::Float32),
Kind::Double => Ok(Self::Float64),
Kind::Half => Ok(Self::Float16),
Kind::BFloat16 => Ok(Self::BFloat16),
_ => Err(invalid(
"optimizer parameters must use Float, Double, Half or BFloat16",
)),
}
}
fn kind(self) -> Kind {
match self {
Self::Float32 => Kind::Float,
Self::Float64 => Kind::Double,
Self::Float16 => Kind::Half,
Self::BFloat16 => Kind::BFloat16,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct TensorState {
shape: Vec<i64>,
dtype: Dtype,
bytes: Vec<u8>,
}
impl TensorState {
fn capture(tensor: &Tensor) -> Result<Self> {
let dtype = Dtype::from_kind(tensor.kind())?;
let size = tensor
.numel()
.checked_mul(dtype.kind().elt_size_in_bytes())
.ok_or_else(|| invalid("tensor byte count overflow"))?;
let mut bytes = Vec::new();
bytes
.try_reserve_exact(size)
.map_err(|_| invalid("cannot reserve checkpoint payload"))?;
bytes.resize(size, 0);
tensor
.f_to_device(Device::Cpu)?
.f_contiguous()?
.f_copy_data_u8(&mut bytes, tensor.numel())?;
if cfg!(target_endian = "big") {
swap_elements(&mut bytes, dtype.kind().elt_size_in_bytes());
}
Ok(Self {
shape: tensor.size(),
dtype,
bytes,
})
}
fn validate(&self, shape: &[i64], dtype: Dtype) -> Result<()> {
if self.shape != shape || self.dtype != dtype {
return Err(invalid("moment shape or dtype differs from its parameter"));
}
let numel = self
.shape
.iter()
.try_fold(1usize, |n, &dimension| {
usize::try_from(dimension)
.ok()
.and_then(|d| n.checked_mul(d))
})
.ok_or_else(|| invalid("invalid checkpoint dimensions"))?;
if numel.checked_mul(dtype.kind().elt_size_in_bytes()) != Some(self.bytes.len()) {
return Err(invalid(
"checkpoint byte count does not match shape and dtype",
));
}
Ok(())
}
fn restore(&self, device: Device) -> Result<Tensor> {
let mut bytes = self.bytes.clone();
if cfg!(target_endian = "big") {
swap_elements(&mut bytes, self.dtype.kind().elt_size_in_bytes());
}
Ok(
Tensor::f_from_data_size(&bytes, &self.shape, self.dtype.kind())?
.f_to_device(device)?,
)
}
}
fn swap_elements(bytes: &mut [u8], width: usize) {
for element in bytes.chunks_exact_mut(width) {
element.reverse();
}
}
#[derive(Debug)]
pub struct Optimizer {
variables: Arc<Mutex<Variables>>,
parameters: BTreeMap<String, Parameter>,
groups: BTreeMap<usize, ParameterGroup>,
algorithm: Algorithm,
default_learning_rate: f64,
default_weight_decay: f64,
completed_steps: u64,
gradient_clear: COptimizer,
id: u64,
}
impl Optimizer {
pub(super) fn new(store: &VarStore, algorithm: Algorithm, lr: f64, wd: f64) -> Result<Self> {
algorithm.validate()?;
validate_non_negative("learning_rate", lr)?;
validate_non_negative("weight_decay", wd)?;
let mut optimizer = Self {
variables: store.variables_.clone(),
parameters: BTreeMap::new(),
groups: BTreeMap::new(),
algorithm,
default_learning_rate: lr,
default_weight_decay: wd,
completed_steps: 0,
gradient_clear: COptimizer::sgd(0., 0., 0., 0., false)?,
id: NEXT_OPTIMIZER_ID.fetch_add(1, Ordering::Relaxed),
};
optimizer.sync_parameters()?;
Ok(optimizer)
}
pub(crate) fn distributed_configuration(&mut self) -> Result<Vec<u8>> {
self.sync_parameters()?;
let parameters = self
.parameters
.iter()
.map(|(name, p)| {
(
name,
p.tensor.size(),
format!("{:?}", p.tensor.kind()),
p.group,
p.step,
)
})
.collect::<Vec<_>>();
serde_json::to_vec(&(
&self.algorithm,
self.default_learning_rate,
self.default_weight_decay,
self.completed_steps,
&self.groups,
parameters,
))
.map_err(|e| invalid(&format!("cannot encode optimizer configuration: {e}")))
}
pub(crate) fn belongs_to(&self, store: &VarStore) -> bool {
Arc::ptr_eq(&self.variables, &store.variables_)
}
pub(crate) fn identity(&self) -> u64 {
self.id
}
fn sync_parameters(&mut self) -> Result<()> {
let variables = self
.variables
.lock()
.map_err(|_| invalid("parameter store lock is poisoned"))?;
if variables.trainable_variables.len() == self.parameters.len() {
return Ok(());
}
if variables.trainable_variables.len() < self.parameters.len() {
return Err(invalid("registered parameters were removed"));
}
let mut additions = Vec::new();
for variable in &variables.trainable_variables {
if !variable.tensor.defined() || variable.tensor.is_sparse() {
return Err(invalid("parameters must be defined dense tensors"));
}
Dtype::from_kind(variable.tensor.kind())?;
let mut names = variables
.named_variables
.iter()
.filter_map(
|(name, tensor)| match tensor.f_is_set_to(&variable.tensor) {
Ok(true) => Some(Ok(name)),
Ok(false) => None,
Err(error) => Some(Err(error)),
},
)
.collect::<std::result::Result<Vec<_>, _>>()?;
names.sort();
let name = names
.first()
.ok_or_else(|| invalid("trainable parameter has no registered name"))?;
if names.len() != 1 {
return Err(invalid(
"aliased parameter registrations require one canonical name",
));
}
if let Some(current) = self.parameters.get(*name) {
if current.group != variable.group
|| !current.tensor.f_is_set_to(&variable.tensor)?
{
return Err(invalid(
"parameter identity or group changed after registration",
));
}
} else {
if additions.iter().any(|(prior, _, _)| prior == *name) {
return Err(invalid("duplicate trainable parameter registration"));
}
additions.push((
(*name).clone(),
variable.group,
variable.tensor.shallow_clone(),
));
}
}
for (name, group, tensor) in additions {
self.gradient_clear.add_parameters(&tensor, 0)?;
self.groups
.entry(group)
.or_insert_with(|| ParameterGroup {
id: group,
learning_rate: self.default_learning_rate,
weight_decay: self.default_weight_decay,
parameters: Vec::new(),
})
.parameters
.push(name.clone());
self.parameters.insert(
name,
Parameter {
tensor,
group,
step: 0,
slots: BTreeMap::new(),
},
);
}
for group in self.groups.values_mut() {
group.parameters.sort();
}
Ok(())
}
pub fn trainable_variables(&self) -> Vec<Tensor> {
self.variables
.lock()
.expect("parameter store lock is poisoned")
.trainable_variables
.iter()
.map(|p| p.tensor.shallow_clone())
.collect()
}
pub fn named_parameters(&self) -> Vec<(String, Tensor)> {
self.parameters
.iter()
.map(|(name, p)| (name.clone(), p.tensor.shallow_clone()))
.collect()
}
pub fn parameter_groups(&self) -> Vec<ParameterGroup> {
self.groups.values().cloned().collect()
}
pub(super) fn apply_group_rates(&mut self, rates: &[(usize, f64)]) -> Result<()> {
self.sync_parameters()?;
if rates.len() != self.groups.len()
|| rates.iter().map(|(id, _)| *id).collect::<Vec<_>>()
!= self.groups.keys().copied().collect::<Vec<_>>()
{
return Err(invalid("scheduler parameter groups changed"));
}
for (_, value) in rates {
validate_non_negative("learning_rate", *value)?;
}
for (id, value) in rates {
self.groups
.get_mut(id)
.expect("validated group")
.learning_rate = *value;
}
Ok(())
}
pub fn set_learning_rate(&mut self, value: f64) -> Result<()> {
validate_non_negative("learning_rate", value)?;
self.sync_parameters()?;
for group in self.groups.values_mut() {
group.learning_rate = value;
}
self.default_learning_rate = value;
Ok(())
}
pub fn set_group_learning_rate(&mut self, id: usize, value: f64) -> Result<()> {
validate_non_negative("learning_rate", value)?;
self.sync_parameters()?;
self.groups
.get_mut(&id)
.ok_or_else(|| invalid("unknown parameter group"))?
.learning_rate = value;
Ok(())
}
pub fn set_group_weight_decay(&mut self, id: usize, value: f64) -> Result<()> {
validate_non_negative("weight_decay", value)?;
self.sync_parameters()?;
self.groups
.get_mut(&id)
.ok_or_else(|| invalid("unknown parameter group"))?
.weight_decay = value;
Ok(())
}
pub fn try_zero_grad(&mut self) -> Result<()> {
self.sync_parameters()?;
self.gradient_clear.zero_grad()?;
Ok(())
}
pub fn zero_grad(&mut self) {
self.try_zero_grad().expect("optimizer zero_grad failed");
}
pub fn step(&mut self) {
self.try_step().expect("optimizer step failed");
}
pub fn try_step(&mut self) -> Result<()> {
self.sync_parameters()?;
let next_step = self
.completed_steps
.checked_add(1)
.filter(|step| *step < u64::MAX)
.ok_or_else(|| invalid("optimizer step counter overflow"))?;
for parameter in self.parameters.values() {
let gradient = parameter.tensor.f_grad()?;
if !gradient.defined() {
continue;
}
validate_gradient(
parameter,
&gradient,
&self.algorithm,
&self.groups[¶meter.group],
)?;
}
no_grad(|| -> Result<()> {
let mut updates = Vec::new();
for (name, parameter) in &self.parameters {
let gradient = parameter.tensor.f_grad()?;
if gradient.defined() {
let step = parameter
.step
.checked_add(1)
.filter(|step| *step < u64::MAX)
.ok_or_else(|| invalid("parameter step counter overflow"))?;
let (value, slots) = update(
&self.algorithm,
parameter,
gradient,
&self.groups[¶meter.group],
step,
)?;
updates.push((name.clone(), value, slots, step));
}
}
for (name, value, slots, step) in updates {
let parameter = self
.parameters
.get_mut(&name)
.expect("staged parameter exists");
parameter.tensor.f_copy_(&value)?;
parameter.slots = slots;
parameter.step = step;
}
self.completed_steps = next_step;
Ok(())
})
}
pub fn backward_step(&mut self, loss: &Tensor) -> Result<()> {
validate_loss(loss)?;
self.try_zero_grad()?;
loss.f_backward()?;
self.try_step()
}
pub fn clip_grad_norm(&self, max: f64) -> Result<()> {
validate_non_negative("max_grad_norm", max)?;
let gradients = self.dense_gradients()?;
if gradients.is_empty() {
return Ok(());
}
let norms = gradients
.iter()
.map(Tensor::f_norm)
.collect::<std::result::Result<Vec<_>, _>>()?;
let total = Tensor::f_stack(&norms, 0)?.f_norm()?.f_double_value(&[])?;
let coefficient = max / (total + 1e-6);
no_grad(|| -> Result<()> {
if coefficient < 1. {
for mut gradient in gradients {
let _ = gradient.f_mul_scalar_(coefficient)?;
}
}
Ok(())
})
}
pub fn clip_grad_value(&self, max: f64) -> Result<()> {
validate_non_negative("max_grad_value", max)?;
let gradients = self.dense_gradients()?;
no_grad(|| -> Result<()> {
for mut gradient in gradients {
let _ = gradient.f_clamp_(-max, max)?;
}
Ok(())
})
}
fn dense_gradients(&self) -> Result<Vec<Tensor>> {
let variables = self
.variables
.lock()
.map_err(|_| invalid("parameter store lock is poisoned"))?;
let mut gradients = Vec::new();
for parameter in &variables.trainable_variables {
let gradient = parameter.tensor.f_grad()?;
if gradient.defined() {
if gradient.is_sparse() {
return Err(invalid("gradient clipping requires dense gradients"));
}
gradients.push(gradient);
}
}
Ok(gradients)
}
pub fn state_dict(&mut self) -> Result<OptimizerState> {
self.sync_parameters()?;
for parameter in self.parameters.values() {
validate_slots(parameter)?;
}
let mut parameters = BTreeMap::new();
for (name, p) in &self.parameters {
let mut slots = BTreeMap::new();
for (slot, tensor) in &p.slots {
slots.insert(slot.clone(), TensorState::capture(tensor)?);
}
parameters.insert(
name.clone(),
ParameterState {
shape: p.tensor.size(),
dtype: Dtype::from_kind(p.tensor.kind())?,
group: p.group,
step: p.step,
slots,
},
);
}
Ok(OptimizerState {
schema_version: SCHEMA,
algorithm: self.algorithm.clone(),
default_learning_rate: self.default_learning_rate,
default_weight_decay: self.default_weight_decay,
completed_steps: self.completed_steps,
groups: self.parameter_groups(),
parameters,
})
}
pub fn load_state_dict(&mut self, state: &OptimizerState) -> Result<()> {
let prepared = self.prepare_state_dict(state)?;
self.apply_prepared_state(prepared);
Ok(())
}
pub(crate) fn prepare_state_dict(
&mut self,
state: &OptimizerState,
) -> Result<PreparedOptimizerState> {
self.sync_parameters()?;
if state.schema_version != SCHEMA || !self.algorithm.same_family(&state.algorithm) {
return Err(invalid("optimizer checkpoint schema or algorithm mismatch"));
}
state.algorithm.validate()?;
validate_non_negative("learning_rate", state.default_learning_rate)?;
validate_non_negative("weight_decay", state.default_weight_decay)?;
if state.completed_steps == u64::MAX {
return Err(invalid("optimizer step counter cannot advance"));
}
let mut groups = BTreeMap::new();
for group in &state.groups {
validate_non_negative("learning_rate", group.learning_rate)?;
validate_non_negative("weight_decay", group.weight_decay)?;
let current = self
.groups
.get(&group.id)
.ok_or_else(|| invalid("checkpoint has unknown parameter group"))?;
if current.parameters != group.parameters
|| groups.insert(group.id, group.clone()).is_some()
{
return Err(invalid(
"checkpoint group membership differs or has duplicates",
));
}
}
if groups.len() != self.groups.len() || state.parameters.len() != self.parameters.len() {
return Err(invalid("checkpoint parameter/group count mismatch"));
}
for (name, parameter) in &self.parameters {
let saved = state
.parameters
.get(name)
.ok_or_else(|| invalid("checkpoint parameter name mismatch"))?;
let dtype = Dtype::from_kind(parameter.tensor.kind())?;
if saved.shape != parameter.tensor.size()
|| saved.dtype != dtype
|| saved.group != parameter.group
{
return Err(invalid(
"checkpoint parameter shape, dtype or group mismatch",
));
}
if saved.step > state.completed_steps || saved.step == u64::MAX {
return Err(invalid("invalid parameter step counter"));
}
let expected: BTreeSet<_> = if saved.step == 0 {
BTreeSet::new()
} else {
state.algorithm.slots().iter().copied().collect()
};
if saved
.slots
.keys()
.map(String::as_str)
.collect::<BTreeSet<_>>()
!= expected
{
return Err(invalid("checkpoint moment names or cardinality mismatch"));
}
for tensor in saved.slots.values() {
tensor.validate(&saved.shape, dtype)?;
}
}
let mut restored = BTreeMap::new();
for (name, saved) in &state.parameters {
let device = self.parameters[name].tensor.device();
let mut slots = BTreeMap::new();
for (slot, tensor) in &saved.slots {
slots.insert(slot.clone(), tensor.restore(device)?);
}
restored.insert(name.clone(), slots);
}
Ok(PreparedOptimizerState {
restored,
steps: state
.parameters
.iter()
.map(|(name, saved)| (name.clone(), saved.step))
.collect(),
groups,
algorithm: state.algorithm.clone(),
default_learning_rate: state.default_learning_rate,
default_weight_decay: state.default_weight_decay,
completed_steps: state.completed_steps,
})
}
pub(crate) fn apply_prepared_state(&mut self, prepared: PreparedOptimizerState) {
for (name, slots) in prepared.restored {
let parameter = self
.parameters
.get_mut(&name)
.expect("prepared optimizer parameter exists");
parameter.slots = slots;
parameter.step = prepared.steps[&name];
}
self.groups = prepared.groups;
self.algorithm = prepared.algorithm;
self.default_learning_rate = prepared.default_learning_rate;
self.default_weight_decay = prepared.default_weight_decay;
self.completed_steps = prepared.completed_steps;
}
}
fn validate_gradient(
p: &Parameter,
gradient: &Tensor,
algorithm: &Algorithm,
group: &ParameterGroup,
) -> Result<()> {
Dtype::from_kind(p.tensor.kind())?;
if gradient.size() != p.tensor.size()
|| gradient.kind() != p.tensor.kind()
|| gradient.device() != p.tensor.device()
{
return Err(invalid(
"gradient shape, dtype or device differs from its parameter",
));
}
if gradient.is_sparse()
&& !matches!(algorithm, Algorithm::Sgd { momentum: 0., .. } if group.weight_decay == 0.)
{
return Err(invalid(
"sparse gradients require SGD without momentum or weight decay",
));
}
if !gradient.is_sparse() {
let _ = gradient.f_contiguous()?;
}
validate_slots(p)
}
fn validate_slots(p: &Parameter) -> Result<()> {
if !p.tensor.defined() {
return Err(invalid("parameter is undefined"));
}
Dtype::from_kind(p.tensor.kind())?;
for slot in p.slots.values() {
if slot.size() != p.tensor.size()
|| slot.kind() != p.tensor.kind()
|| slot.device() != p.tensor.device()
{
return Err(invalid(
"parameter was moved or reshaped without matching optimizer state",
));
}
}
Ok(())
}
fn initial(p: &Parameter, name: &str, value: f64) -> Result<Tensor> {
Ok(match p.slots.get(name) {
Some(tensor) => tensor.shallow_clone(),
None => p.tensor.f_zeros_like()?.f_add_scalar(value)?,
})
}
fn update(
algorithm: &Algorithm,
p: &Parameter,
mut gradient: Tensor,
group: &ParameterGroup,
step: u64,
) -> Result<(Tensor, BTreeMap<String, Tensor>)> {
let mut value = p.tensor.shallow_clone();
let mut slots = BTreeMap::new();
let lr = group.learning_rate;
let decay = group.weight_decay;
if decay != 0. {
if matches!(
algorithm,
Algorithm::Adam {
decoupled: true,
..
}
) {
value = value.f_mul_scalar(1. - lr * decay)?;
} else {
gradient = gradient.f_add(&value.f_mul_scalar(decay)?)?;
}
}
let direction = match *algorithm {
Algorithm::Sgd {
momentum,
dampening,
nesterov,
} => {
if momentum == 0. {
gradient
} else {
let buffer = if p.step == 0 {
gradient.f_detach_copy()?
} else {
initial(p, "momentum_buffer", 0.)?
.f_mul_scalar(momentum)?
.f_add(&gradient.f_mul_scalar(1. - dampening)?)?
};
let direction = if nesterov {
gradient.f_add(&buffer.f_mul_scalar(momentum)?)?
} else {
buffer.shallow_clone()
};
slots.insert("momentum_buffer".into(), buffer);
direction
}
}
Algorithm::Adam {
beta1,
beta2,
eps,
amsgrad,
..
} => {
let m = initial(p, "exp_avg", 0.)?.f_lerp(&gradient, 1. - beta1)?;
let v = initial(p, "exp_avg_sq", 0.)?
.f_mul_scalar(beta2)?
.f_add(&gradient.f_square()?.f_mul_scalar(1. - beta2)?)?;
let mut denominator = v.shallow_clone();
if amsgrad {
denominator = initial(p, "max_exp_avg_sq", 0.)?.f_maximum(&v)?;
slots.insert("max_exp_avg_sq".into(), denominator.shallow_clone());
}
denominator = denominator
.f_sqrt()?
.f_div_scalar((1. - beta2.powf(step as f64)).sqrt())?
.f_add_scalar(eps)?;
let direction = m
.f_div(&denominator)?
.f_div_scalar(1. - beta1.powf(step as f64))?;
slots.insert("exp_avg".into(), m);
slots.insert("exp_avg_sq".into(), v);
direction
}
Algorithm::RmsProp {
alpha,
eps,
momentum,
centered,
} => {
let square = initial(p, "square_avg", 0.)?
.f_mul_scalar(alpha)?
.f_add(&gradient.f_square()?.f_mul_scalar(1. - alpha)?)?;
let mut avg = square.shallow_clone();
if centered {
let mean = initial(p, "grad_avg", 0.)?.f_lerp(&gradient, 1. - alpha)?;
avg = avg.f_sub(&mean.f_square()?)?;
slots.insert("grad_avg".into(), mean);
}
let direction = gradient.f_div(&avg.f_sqrt()?.f_add_scalar(eps)?)?;
slots.insert("square_avg".into(), square);
if momentum != 0. {
let buffer = initial(p, "momentum_buffer", 0.)?
.f_mul_scalar(momentum)?
.f_add(&direction)?;
slots.insert("momentum_buffer".into(), buffer.shallow_clone());
buffer
} else {
direction
}
}
Algorithm::Adagrad {
lr_decay,
initial_accumulator,
eps,
} => {
let sum = initial(p, "sum", initial_accumulator)?.f_add(&gradient.f_square()?)?;
let direction = gradient
.f_div(&sum.f_sqrt()?.f_add_scalar(eps)?)?
.f_div_scalar(1. + (step - 1) as f64 * lr_decay)?;
slots.insert("sum".into(), sum);
direction
}
Algorithm::Adadelta { rho, eps } => {
let square = initial(p, "square_avg", 0.)?
.f_mul_scalar(rho)?
.f_add(&gradient.f_square()?.f_mul_scalar(1. - rho)?)?;
let acc = initial(p, "acc_delta", 0.)?;
let delta = acc
.f_add_scalar(eps)?
.f_sqrt()?
.f_div(&square.f_add_scalar(eps)?.f_sqrt()?)?
.f_mul(&gradient)?;
slots.insert(
"acc_delta".into(),
acc.f_mul_scalar(rho)?
.f_add(&delta.f_square()?.f_mul_scalar(1. - rho)?)?,
);
slots.insert("square_avg".into(), square);
delta
}
Algorithm::Adamax { beta1, beta2, eps } => {
let m = initial(p, "exp_avg", 0.)?.f_lerp(&gradient, 1. - beta1)?;
let infinity = initial(p, "exp_inf", 0.)?
.f_mul_scalar(beta2)?
.f_maximum(&gradient.f_abs()?.f_add_scalar(eps)?)?;
let direction = m
.f_div(&infinity)?
.f_div_scalar(1. - beta1.powf(step as f64))?;
slots.insert("exp_avg".into(), m);
slots.insert("exp_inf".into(), infinity);
direction
}
};
Ok((value.f_add(&direction.f_mul_scalar(-lr)?)?, slots))
}
pub(super) fn invalid(reason: &str) -> RustTorchError {
RustTorchError::InvalidConfiguration {
field: "optimizer state",
reason: reason.to_owned(),
}
}
pub(crate) struct PreparedOptimizerState {
restored: BTreeMap<String, BTreeMap<String, Tensor>>,
steps: BTreeMap<String, u64>,
groups: BTreeMap<usize, ParameterGroup>,
algorithm: Algorithm,
default_learning_rate: f64,
default_weight_decay: f64,
completed_steps: u64,
}
impl OptimizerState {
fn validate_portable(&self) -> Result<()> {
if self.schema_version != SCHEMA || self.completed_steps == u64::MAX {
return Err(invalid("invalid optimizer state schema or step"));
}
self.algorithm.validate()?;
validate_non_negative("learning_rate", self.default_learning_rate)?;
validate_non_negative("weight_decay", self.default_weight_decay)?;
let mut names = BTreeSet::new();
let mut ids = BTreeSet::new();
for group in &self.groups {
validate_non_negative("learning_rate", group.learning_rate)?;
validate_non_negative("weight_decay", group.weight_decay)?;
if !ids.insert(group.id) || group.parameters.windows(2).any(|pair| pair[0] >= pair[1]) {
return Err(invalid(
"duplicate optimizer group or unordered parameter names",
));
}
for name in &group.parameters {
let p = self
.parameters
.get(name)
.ok_or_else(|| invalid("optimizer group names a missing parameter"))?;
if p.group != group.id || !names.insert(name) {
return Err(invalid("optimizer parameter group mismatch"));
}
}
}
if names.len() != self.parameters.len() {
return Err(invalid("optimizer parameter is absent from groups"));
}
for p in self.parameters.values() {
elements(&p.shape)?;
if p.step > self.completed_steps || p.step == u64::MAX {
return Err(invalid("invalid optimizer parameter step"));
}
let expected: BTreeSet<_> = if p.step == 0 {
BTreeSet::new()
} else {
self.algorithm.slots().iter().copied().collect()
};
if p.slots.keys().map(String::as_str).collect::<BTreeSet<_>>() != expected {
return Err(invalid("optimizer slot names mismatch"));
}
for slot in p.slots.values() {
slot.validate(&p.shape, p.dtype)?;
}
}
Ok(())
}
pub(crate) fn parameter_specs(&self) -> Result<BTreeMap<String, (Vec<i64>, Kind)>> {
self.validate_portable()?;
Ok(self
.parameters
.iter()
.map(|(name, p)| (name.clone(), (p.shape.clone(), p.dtype.kind())))
.collect())
}
pub(crate) fn shard(&self, rank: usize, world_size: usize) -> Result<Self> {
self.validate_portable()?;
if world_size == 0 || rank >= world_size {
return Err(invalid("invalid optimizer shard rank/world size"));
}
let mut result = self.clone();
for p in result.parameters.values_mut() {
let count = elements(&p.shape)?;
let chunk = count.div_ceil(world_size);
let start = rank
.checked_mul(chunk)
.ok_or_else(|| invalid("optimizer shard offset overflow"))?
.min(count);
let end = start.saturating_add(chunk).min(count);
let width = p.dtype.kind().elt_size_in_bytes();
for slot in p.slots.values_mut() {
let mut bytes = vec![
0;
chunk.checked_mul(width).ok_or_else(|| invalid(
"optimizer shard byte count overflow"
))?
];
bytes[..(end - start) * width]
.copy_from_slice(&slot.bytes[start * width..end * width]);
slot.bytes = bytes;
slot.shape = vec![
i64::try_from(chunk).map_err(|_| invalid("optimizer shard shape overflow"))?,
];
}
p.shape =
vec![i64::try_from(chunk).map_err(|_| invalid("optimizer shard shape overflow"))?];
}
Ok(result)
}
pub(crate) fn consolidate_shards(
states: &[Self],
parameter_shapes: &BTreeMap<String, Vec<i64>>,
) -> Result<Self> {
let first = states
.first()
.ok_or_else(|| invalid("optimizer shard set is empty"))?;
for state in states {
state.validate_portable()?;
if state.algorithm != first.algorithm
|| state.default_learning_rate != first.default_learning_rate
|| state.default_weight_decay != first.default_weight_decay
|| state.completed_steps != first.completed_steps
|| state.groups != first.groups
|| state.parameters.keys().ne(first.parameters.keys())
{
return Err(invalid("optimizer shard configuration or steps differ"));
}
}
if parameter_shapes.keys().ne(first.parameters.keys()) {
return Err(invalid("optimizer shard names differ from model"));
}
let mut result = first.clone();
for (name, p) in &mut result.parameters {
let shape = ¶meter_shapes[name];
let count = elements(shape)?;
let chunk = count.div_ceil(states.len());
let chunk_i64 =
i64::try_from(chunk).map_err(|_| invalid("optimizer shard size overflow"))?;
for state in states {
let value = &state.parameters[name];
if value.shape != [chunk_i64]
|| value.dtype != p.dtype
|| value.group != p.group
|| value.step != p.step
|| value.slots.keys().ne(p.slots.keys())
{
return Err(invalid("optimizer shard parameter metadata differs"));
}
}
for (slot_name, slot) in &mut p.slots {
let width = p.dtype.kind().elt_size_in_bytes();
let length = count
.checked_mul(width)
.ok_or_else(|| invalid("optimizer checkpoint byte count overflow"))?;
let mut bytes = Vec::new();
bytes
.try_reserve_exact(length)
.map_err(|_| invalid("cannot allocate consolidated optimizer state"))?;
for state in states {
let source = &state.parameters[name].slots[slot_name].bytes;
let remaining = length - bytes.len();
bytes.extend_from_slice(&source[..remaining.min(source.len())]);
}
slot.bytes = bytes;
slot.shape = shape.clone();
}
p.shape = shape.clone();
}
result.validate_portable()?;
Ok(result)
}
}
fn elements(shape: &[i64]) -> Result<usize> {
if shape.len() > 64 {
return Err(invalid("optimizer tensor rank exceeds limit"));
}
shape
.iter()
.try_fold(1usize, |n, &d| {
usize::try_from(d).ok().and_then(|d| n.checked_mul(d))
})
.ok_or_else(|| invalid("optimizer tensor dimensions are negative or overflow"))
}