use crate::{Kind, Result, RustTorchError, Tensor};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct GradOptions {
pub retain_graph: bool,
pub create_graph: bool,
}
pub fn grad(loss: &Tensor, inputs: &[&Tensor], options: GradOptions) -> Result<Vec<Tensor>> {
scalar(loss)?;
if inputs.is_empty() {
return Err(invalid("inputs", "at least one input is required"));
}
for input in inputs {
tracked(input)?;
}
if !loss.requires_grad() {
return Err(invalid("loss", "the loss has no recorded gradient graph"));
}
let values = Tensor::f_run_backward(
&[loss],
inputs,
options.retain_graph || options.create_graph,
options.create_graph,
)?;
if values.iter().any(|value| !value.defined()) {
return Err(invalid(
"inputs",
"an input does not participate in the loss",
));
}
Ok(values)
}
pub fn vjp(
output: &Tensor,
input: &Tensor,
cotangent: &Tensor,
options: GradOptions,
) -> Result<Tensor> {
real(output)?;
same(output, cotangent, "cotangent")?;
tracked(input)?;
let weighted = output.f_mul(&cotangent.f_detach()?)?.f_sum(output.kind())?;
Ok(grad(&weighted, &[input], options)?.remove(0))
}
pub fn jvp(
function: impl FnOnce(&Tensor) -> Result<Tensor>,
input: &Tensor,
tangent: &Tensor,
create_graph: bool,
) -> Result<(Tensor, Tensor)> {
same(input, tangent, "tangent")?;
let x = prepare(input, create_graph)?;
let output = function(&x)?;
real(&output)?;
let seed = output.f_zeros_like()?.f_set_requires_grad(true)?;
let weighted = output.f_mul(&seed)?.f_sum(output.kind())?;
let first = derivative_or_zero(&weighted, &x, true, true)?;
let contracted = first.f_mul(&tangent.f_detach()?)?.f_sum(x.kind())?;
let result = derivative_or_zero(&contracted, &seed, create_graph, create_graph)?;
if create_graph {
Ok((output, result))
} else {
Ok((output.f_detach()?, result.f_detach()?))
}
}
pub fn jacobian(
function: impl FnOnce(&Tensor) -> Result<Tensor>,
input: &Tensor,
create_graph: bool,
) -> Result<Tensor> {
let x = prepare(input, create_graph)?;
let output = function(&x)?;
real(&output)?;
let mut shape = output.size();
shape.extend(x.size());
if output.numel() == 0 {
return Ok(Tensor::f_zeros(shape.as_slice(), (x.kind(), x.device()))?);
}
let flat = output.f_reshape([-1])?;
let rows = (0..output.numel())
.map(|index| {
let item = flat.f_select(0, index as i64)?;
derivative_or_zero(
&item,
&x,
index + 1 < output.numel() || create_graph,
create_graph,
)
})
.collect::<Result<Vec<_>>>()?;
let result = Tensor::f_stack(&rows, 0)?.f_reshape(shape.as_slice())?;
if create_graph {
Ok(result)
} else {
Ok(result.f_detach()?)
}
}
pub fn hessian(
function: impl FnOnce(&Tensor) -> Result<Tensor>,
input: &Tensor,
create_graph: bool,
) -> Result<Tensor> {
jacobian(
|x| {
let loss = function(x)?;
scalar(&loss)?;
derivative_or_zero(&loss, x, true, true)
},
input,
create_graph,
)
}
pub fn with_surrogate_gradient(value: &Tensor, surrogate: &Tensor) -> Result<Tensor> {
same(value, surrogate, "surrogate")?;
if surrogate.f_isfinite()?.f_all()?.f_int64_value(&[])? == 0 {
return Err(invalid(
"surrogate",
"must be finite to preserve the forward value",
));
}
Ok(value
.f_detach()?
.f_add(&surrogate.f_sub(&surrogate.f_detach()?)?)?)
}
fn derivative_or_zero(loss: &Tensor, input: &Tensor, keep: bool, create: bool) -> Result<Tensor> {
if !loss.requires_grad() {
return Ok(input.f_zeros_like()?);
}
let gradient = Tensor::f_run_backward(&[loss], &[input], keep, create)?.remove(0);
if gradient.defined() {
Ok(gradient)
} else {
Ok(input.f_zeros_like()?)
}
}
fn prepare(input: &Tensor, create_graph: bool) -> Result<Tensor> {
real(input)?;
if create_graph && input.requires_grad() {
Ok(input.shallow_clone())
} else {
Ok(input.f_detach()?.f_set_requires_grad(true)?)
}
}
fn tracked(input: &Tensor) -> Result<()> {
real(input)?;
if !input.requires_grad() {
return Err(invalid(
"input",
"must require gradients before the forward pass",
));
}
Ok(())
}
fn scalar(input: &Tensor) -> Result<()> {
real(input)?;
if !input.size().is_empty() {
return Err(invalid("loss", "expected a scalar; reduce the loss first"));
}
Ok(())
}
fn real(input: &Tensor) -> Result<()> {
if !input.defined() {
return Err(invalid("tensor", "must be defined"));
}
if !matches!(
input.kind(),
Kind::Float | Kind::Double | Kind::Half | Kind::BFloat16
) || input.is_sparse()
{
return Err(invalid(
"tensor",
"expected a dense real floating-point tensor",
));
}
let _ = input.f_as_strided([0], [1], None)?;
Ok(())
}
fn same(input: &Tensor, other: &Tensor, name: &'static str) -> Result<()> {
real(input)?;
real(other)?;
if input.size() != other.size()
|| input.kind() != other.kind()
|| input.device() != other.device()
{
return Err(invalid(name, "shape, dtype and device must match"));
}
Ok(())
}
fn invalid(field: &'static str, reason: &str) -> RustTorchError {
RustTorchError::InvalidConfiguration {
field,
reason: reason.into(),
}
}