dfdx 0.13.0

Ergonomic auto differentiation in Rust, with pytorch like apis.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

use crate::tensor_ops::cpu_kernels::UnaryDerivative;
#[cfg(feature = "f16")]
use half::f16;
use libm::{erf, erff};
use num_traits::{Float, FloatConst};

trait Erf {
    fn erf(self) -> Self;
}

#[cfg(feature = "f16")]
impl Erf for crate::dtypes::AMP<f16> {
    fn erf(self) -> Self {
        crate::dtypes::AMP(f16::from_f32(erff(self.0.to_f32())))
    }
}

#[cfg(feature = "f16")]
impl Erf for f16 {
    fn erf(self) -> Self {
        f16::from_f32(erff(self.to_f32()))
    }
}

impl Erf for f64 {
    fn erf(self) -> Self {
        erf(self)
    }
}

impl Erf for f32 {
    fn erf(self) -> Self {
        erff(self)
    }
}

impl<F: Float + FloatConst + Erf> UnaryDerivative<F> for super::AccurateGeLUKernelOp {
    const DF_USES_FX: bool = false;
    const HAS_CONST_DF: bool = false;
    #[inline(always)]
    fn f(&self, &x: &F) -> F {
        let alpha = F::FRAC_1_SQRT_2();
        F::from(0.5).unwrap() * x * (F::one() + (x * alpha).erf())
    }

    #[inline(always)]
    fn df(&self, &x: &F) -> F {
        let half = F::from(0.5).unwrap();
        let alpha = F::FRAC_1_SQRT_2();
        let x_sq = x * x;
        let normal_dist = F::FRAC_2_SQRT_PI() * (F::from(0.5).unwrap() * x_sq.neg()).exp();

        let left = half * x;
        let right = F::one() + (alpha * x).erf();

        let left_derivative = half * right;

        let right_derivative = left * normal_dist;

        left_derivative + right_derivative
    }
}