#![forbid(unsafe_code)]
pub trait Differentiable {
type Tangent;
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct NoTangent;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct ZeroTangent;
impl Differentiable for f32 {
type Tangent = f32;
}
impl Differentiable for f64 {
type Tangent = f64;
}
impl Differentiable for () {
type Tangent = NoTangent;
}
impl<A, B> Differentiable for (A, B)
where
A: Differentiable,
B: Differentiable,
{
type Tangent = (A::Tangent, B::Tangent);
}
impl<T, const N: usize> Differentiable for [T; N]
where
T: Differentiable,
{
type Tangent = [T::Tangent; N];
}
pub trait JvpRule<Args>
where
Args: Differentiable,
{
type Output: Differentiable;
fn jvp(
&self,
args: &Args,
tangent: &Args::Tangent,
) -> (Self::Output, <Self::Output as Differentiable>::Tangent);
}
pub trait Pullback<OutputTangent, InputTangent> {
fn apply(self, cotangent: OutputTangent) -> InputTangent;
}
impl<F, OutputTangent, InputTangent> Pullback<OutputTangent, InputTangent> for F
where
F: FnOnce(OutputTangent) -> InputTangent,
{
fn apply(self, cotangent: OutputTangent) -> InputTangent {
self(cotangent)
}
}
pub trait VjpRule<Args>
where
Args: Differentiable,
{
type Output: Differentiable;
type Pullback<'a>: Pullback<<Self::Output as Differentiable>::Tangent, Args::Tangent>
where
Self: 'a,
Args: 'a;
fn vjp<'a>(&'a self, args: &'a Args) -> (Self::Output, Self::Pullback<'a>);
}
pub mod testing {
#[must_use]
pub fn central_difference<F>(f: F, x: f64, direction: f64, step: f64) -> f64
where
F: Fn(f64) -> f64,
{
assert!(step > 0.0, "finite-difference step must be positive");
(f(x + step * direction) - f(x - step * direction)) / (2.0 * step)
}
pub fn assert_close(actual: f64, expected: f64, relative: f64, absolute: f64) {
let tolerance = absolute + relative * actual.abs().max(expected.abs());
assert!(
(actual - expected).abs() <= tolerance,
"values differ: actual={actual}, expected={expected}, tolerance={tolerance}"
);
}
pub fn assert_scalar_adjoint_identity(
jvp: f64,
output_cotangent: f64,
input_tangent: f64,
input_cotangent: f64,
tolerance: f64,
) {
assert_close(
jvp * output_cotangent,
input_tangent * input_cotangent,
tolerance,
tolerance,
);
}
}
#[cfg(test)]
mod tests {
use super::{JvpRule, Pullback, VjpRule};
use crate::testing::{assert_close, assert_scalar_adjoint_identity, central_difference};
struct Square;
struct SquarePullback {
x: f64,
}
impl Pullback<f64, f64> for SquarePullback {
fn apply(self, cotangent: f64) -> f64 {
2.0 * self.x * cotangent
}
}
impl JvpRule<f64> for Square {
type Output = f64;
fn jvp(&self, x: &f64, tangent: &f64) -> (f64, f64) {
(x * x, 2.0 * x * tangent)
}
}
impl VjpRule<f64> for Square {
type Output = f64;
type Pullback<'a> = SquarePullback;
fn vjp<'a>(&'a self, x: &'a f64) -> (f64, Self::Pullback<'a>) {
(x * x, SquarePullback { x: *x })
}
}
#[test]
fn jvp_matches_finite_difference() {
let x = 3.0;
let direction = -0.25;
let (_, jvp) = Square.jvp(&x, &direction);
let numerical = central_difference(|value| value * value, x, direction, 1.0e-5);
assert_close(jvp, numerical, 1.0e-9, 1.0e-9);
}
#[test]
fn vjp_satisfies_adjoint_identity() {
let x = 3.0;
let input_tangent = -0.25;
let output_cotangent = 1.75;
let (_, jvp) = Square.jvp(&x, &input_tangent);
let (_, pullback) = Square.vjp(&x);
let input_cotangent = pullback.apply(output_cotangent);
assert_scalar_adjoint_identity(
jvp,
output_cotangent,
input_tangent,
input_cotangent,
1.0e-12,
);
}
}