use crate::autograd::Gradients;
use crate::err::Result;
use crate::tensor::backend::{AutogradBackend, VariableBackend};
pub trait Optimizer<B: VariableBackend + AutogradBackend> {
fn step(&mut self, grads: &Gradients<B>) -> Result<()>;
}
pub trait ScaledOptimizer<B: VariableBackend + AutogradBackend>: Optimizer<B> {
fn step_scaled(
&mut self,
grads: &mut Gradients<B>,
scaler: &mut crate::exec::LossScaleState,
) -> Result<bool>;
}
use crate::backend_authoring::{Capabilities, Execute};
use crate::dist::Local;
use crate::err::Error;
use crate::exec::request::TensorHandle;
use crate::exec::{
catalog::{ClampAttributes, NoAttributes, ScalarAttributes, op},
dispatch,
};
use crate::shapes::Dyn;
use crate::shapes::{ShapeBuf, ShapeValue};
use crate::tensor::dtype::DType;
pub trait OptimizerBackend<K: DType>: VariableBackend {
fn optimizer_add(lhs: &Self::Storage<K>, rhs: &Self::Storage<K>) -> Result<Self::Storage<K>>;
fn optimizer_sub(lhs: &Self::Storage<K>, rhs: &Self::Storage<K>) -> Result<Self::Storage<K>>;
fn optimizer_mul(lhs: &Self::Storage<K>, rhs: &Self::Storage<K>) -> Result<Self::Storage<K>>;
fn optimizer_div(lhs: &Self::Storage<K>, rhs: &Self::Storage<K>) -> Result<Self::Storage<K>>;
fn optimizer_sqrt(storage: &Self::Storage<K>) -> Result<Self::Storage<K>>;
fn optimizer_mul_scalar(storage: &Self::Storage<K>, value: f64) -> Result<Self::Storage<K>>;
fn optimizer_add_scalar(storage: &Self::Storage<K>, value: f64) -> Result<Self::Storage<K>>;
}
impl<B, K: DType> OptimizerBackend<K> for B
where
B: VariableBackend
+ Capabilities
+ Execute<op::Add>
+ Execute<op::Sub>
+ Execute<op::Mul>
+ Execute<op::Div>
+ Execute<op::Sqrt>
+ Execute<op::MulScalar>
+ Execute<op::AddScalar>,
<B as Execute<op::Add>>::Output: Into<B::Storage<K>>,
<B as Execute<op::Sub>>::Output: Into<B::Storage<K>>,
<B as Execute<op::Mul>>::Output: Into<B::Storage<K>>,
<B as Execute<op::Div>>::Output: Into<B::Storage<K>>,
<B as Execute<op::Sqrt>>::Output: Into<B::Storage<K>>,
<B as Execute<op::MulScalar>>::Output: Into<B::Storage<K>>,
<B as Execute<op::AddScalar>>::Output: Into<B::Storage<K>>,
{
fn optimizer_add(lhs: &B::Storage<K>, rhs: &B::Storage<K>) -> Result<B::Storage<K>> {
execute_binary::<op::Add, B, K>(lhs, rhs)
}
fn optimizer_sub(lhs: &B::Storage<K>, rhs: &B::Storage<K>) -> Result<B::Storage<K>> {
execute_binary::<op::Sub, B, K>(lhs, rhs)
}
fn optimizer_mul(lhs: &B::Storage<K>, rhs: &B::Storage<K>) -> Result<B::Storage<K>> {
execute_binary::<op::Mul, B, K>(lhs, rhs)
}
fn optimizer_div(lhs: &B::Storage<K>, rhs: &B::Storage<K>) -> Result<B::Storage<K>> {
execute_binary::<op::Div, B, K>(lhs, rhs)
}
fn optimizer_sqrt(storage: &B::Storage<K>) -> Result<B::Storage<K>> {
execute_unary::<op::Sqrt, B, K>(storage)
}
fn optimizer_mul_scalar(storage: &B::Storage<K>, value: f64) -> Result<B::Storage<K>> {
execute_scalar::<op::MulScalar, B, K>(storage, value)
}
fn optimizer_add_scalar(storage: &B::Storage<K>, value: f64) -> Result<B::Storage<K>> {
execute_scalar::<op::AddScalar, B, K>(storage, value)
}
}
pub trait ValueClippingBackend<K: DType>: OptimizerBackend<K> {
fn optimizer_clamp(storage: &Self::Storage<K>, min: f64, max: f64) -> Result<Self::Storage<K>>;
}
impl<B, K: DType> ValueClippingBackend<K> for B
where
B: OptimizerBackend<K> + Capabilities + Execute<op::Clamp>,
<B as Execute<op::Clamp>>::Output: Into<B::Storage<K>>,
{
fn optimizer_clamp(storage: &B::Storage<K>, min: f64, max: f64) -> Result<B::Storage<K>> {
execute_clamp::<op::Clamp, B, K>(storage, min, max)
}
}
fn execute_binary<O, B, K>(lhs: &B::Storage<K>, rhs: &B::Storage<K>) -> Result<B::Storage<K>>
where
O: crate::exec::catalog::Operation<Attributes = NoAttributes>,
B: VariableBackend + Capabilities + Execute<O>,
K: DType,
<B as Execute<O>>::Output: Into<B::Storage<K>>,
{
let expected =
ShapeValue::<Dyn>::try_new(ShapeBuf::from_slice(&B::shape(lhs))).map_err(Error::Shape)?;
let inputs = [
TensorHandle::from_storage::<B, K, Local>(lhs),
TensorHandle::from_storage::<B, K, Local>(rhs),
];
let context = crate::exec::ExecutionContext::from_scope(B::default())
.with_grad_mode(crate::exec::GradMode::Disabled);
dispatch::execute_shaped::<O, B, Dyn>(&context, NoAttributes, &inputs, &expected)
.map(Into::into)
.map_err(Error::from)
}
fn execute_unary<O, B, K>(storage: &B::Storage<K>) -> Result<B::Storage<K>>
where
O: crate::exec::catalog::Operation<Attributes = NoAttributes>,
B: VariableBackend + Capabilities + Execute<O>,
K: DType,
<B as Execute<O>>::Output: Into<B::Storage<K>>,
{
let expected = ShapeValue::<Dyn>::try_new(ShapeBuf::from_slice(&B::shape(storage)))
.map_err(Error::Shape)?;
let input = TensorHandle::from_storage::<B, K, Local>(storage);
let context = crate::exec::ExecutionContext::from_scope(B::default())
.with_grad_mode(crate::exec::GradMode::Disabled);
dispatch::execute_shaped::<O, B, Dyn>(&context, NoAttributes, &[input], &expected)
.map(Into::into)
.map_err(Error::from)
}
fn execute_scalar<O, B, K>(storage: &B::Storage<K>, value: f64) -> Result<B::Storage<K>>
where
O: crate::exec::catalog::Operation<Attributes = ScalarAttributes>,
B: VariableBackend + Capabilities + Execute<O>,
K: DType,
<B as Execute<O>>::Output: Into<B::Storage<K>>,
{
let expected = ShapeValue::<Dyn>::try_new(ShapeBuf::from_slice(&B::shape(storage)))
.map_err(Error::Shape)?;
let input = TensorHandle::from_storage::<B, K, Local>(storage);
let context = crate::exec::ExecutionContext::from_scope(B::default())
.with_grad_mode(crate::exec::GradMode::Disabled);
dispatch::execute_shaped::<O, B, Dyn>(&context, ScalarAttributes { value }, &[input], &expected)
.map(Into::into)
.map_err(Error::from)
}
fn execute_clamp<O, B, K>(storage: &B::Storage<K>, min: f64, max: f64) -> Result<B::Storage<K>>
where
O: crate::exec::catalog::Operation<Attributes = ClampAttributes>,
B: VariableBackend + Capabilities + Execute<O>,
K: DType,
<B as Execute<O>>::Output: Into<B::Storage<K>>,
{
let expected = ShapeValue::<Dyn>::try_new(ShapeBuf::from_slice(&B::shape(storage)))
.map_err(Error::Shape)?;
let input = TensorHandle::from_storage::<B, K, Local>(storage);
let context = crate::exec::ExecutionContext::from_scope(B::default())
.with_grad_mode(crate::exec::GradMode::Disabled);
dispatch::execute_shaped::<O, B, Dyn>(
&context,
ClampAttributes { min, max },
&[input],
&expected,
)
.map(Into::into)
.map_err(Error::from)
}