use super::contract::UnaryOp;
use crate::domain::RealScalar;
use eunomia::NumericElement;
#[must_use]
fn stable_sigmoid<T: RealScalar>(x: T) -> T {
let zero = <T as NumericElement>::ZERO;
let one = <T as NumericElement>::ONE;
if x >= zero {
one / (one + (-x).exp())
} else {
let e = x.exp();
e / (one + e)
}
}
#[must_use]
fn softplus_value<T: RealScalar>(x: T) -> T {
x.max(<T as NumericElement>::ZERO) + (-<T as NumericElement>::abs(x)).exp().ln_1p()
}
#[must_use]
fn is_infinite<T: RealScalar>(x: T) -> bool {
!x.is_finite() && !x.is_nan()
}
fn clamp_scalar<T: RealScalar>(v: T, lo: T, hi: T) -> T {
if v < lo {
lo
} else if v > hi {
hi
} else {
v
}
}
fn gelu_tanh_arg<T: RealScalar>(x: T) -> T {
let c0 = T::from_f64(0.797_884_560_802_865_4);
let c1 = T::from_f64(0.044715);
c0 * (x + c1 * x * x * x)
}
const SILU_GRAD_CROSSOVER: f64 = 2.63;
const GELU_TANH_GRAD_CROSSOVER: f64 = 2.47;
const MISH_GRAD_CROSSOVER: f64 = 1.55;
const TANH_GRAD_CROSSOVER: f64 = 0.75;
#[derive(Clone, Copy, Debug, Default)]
pub struct ReluOp;
impl<T: RealScalar> UnaryOp<T> for ReluOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, x: T) -> T {
if x.is_nan() {
x
} else {
x.max(<T as NumericElement>::ZERO)
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct ReluGradOp;
impl<T: RealScalar> UnaryOp<T> for ReluGradOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, x: T) -> T {
if x.is_nan() {
x
} else if x > <T as NumericElement>::ZERO {
<T as NumericElement>::ONE
} else {
<T as NumericElement>::ZERO
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct EluOp;
impl<T: RealScalar> UnaryOp<T> for EluOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if x >= <T as NumericElement>::ZERO {
x
} else {
x.exp_m1()
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct EluGradOp;
impl<T: RealScalar> UnaryOp<T> for EluGradOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if x >= <T as NumericElement>::ZERO {
<T as NumericElement>::ONE
} else {
x.exp()
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct SigmoidOp;
impl<T: RealScalar> UnaryOp<T> for SigmoidOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
stable_sigmoid(x)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct SigmoidGradOp;
impl<T: RealScalar> UnaryOp<T> for SigmoidGradOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, y: T) -> T {
y * (<T as NumericElement>::ONE - y)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct TanhGradOp;
impl<T: RealScalar> UnaryOp<T> for TanhGradOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, y: T) -> T {
let one = <T as NumericElement>::ONE;
if <T as NumericElement>::abs(y) <= T::from_f64(TANH_GRAD_CROSSOVER) {
one - y * y
} else {
(one - y) * (one + y)
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct SoftplusOp;
impl<T: RealScalar> UnaryOp<T> for SoftplusOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
softplus_value(x)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct SoftplusGradOp;
impl<T: RealScalar> UnaryOp<T> for SoftplusGradOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
stable_sigmoid(x)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct SoftsignOp;
impl<T: RealScalar> UnaryOp<T> for SoftsignOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, x: T) -> T {
if is_infinite(x) {
return if x > <T as NumericElement>::ZERO {
<T as NumericElement>::ONE
} else {
-<T as NumericElement>::ONE
};
}
x / (<T as NumericElement>::ONE + <T as NumericElement>::abs(x))
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct SoftsignGradOp;
impl<T: RealScalar> UnaryOp<T> for SoftsignGradOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, x: T) -> T {
let denom = <T as NumericElement>::ONE + <T as NumericElement>::abs(x);
<T as NumericElement>::ONE / (denom * denom)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct HardsigmoidOp;
impl<T: RealScalar> UnaryOp<T> for HardsigmoidOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, x: T) -> T {
if x.is_nan() {
return x;
}
let zero = <T as NumericElement>::ZERO;
let one = <T as NumericElement>::ONE;
let six = T::from_f64(6.0);
let half = T::from_f64(0.5);
clamp_scalar(x / six + half, zero, one)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct HardsigmoidGradOp;
impl<T: RealScalar> UnaryOp<T> for HardsigmoidGradOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, x: T) -> T {
if x.is_nan() {
return x;
}
let three = T::from_f64(3.0);
if x > -three && x < three {
<T as NumericElement>::ONE / T::from_f64(6.0)
} else {
<T as NumericElement>::ZERO
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct HardswishOp;
impl<T: RealScalar> UnaryOp<T> for HardswishOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, x: T) -> T {
let zero = <T as NumericElement>::ZERO;
let three = T::from_f64(3.0);
if x <= -three {
return -zero;
}
let six = T::from_f64(6.0);
x * (clamp_scalar(x + three, zero, six) / six)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct HardswishGradOp;
impl<T: RealScalar> UnaryOp<T> for HardswishGradOp {
const COMPUTE_BOUND: bool = false;
#[inline(always)]
fn apply(&self, x: T) -> T {
if x.is_nan() {
return x;
}
let one = <T as NumericElement>::ONE;
let two = one + one;
let three = two + one;
let six = T::from_f64(6.0);
if x >= three {
one
} else if x > -three {
(two * x + three) / six
} else {
<T as NumericElement>::ZERO
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct GeluOp;
impl<T: RealScalar> UnaryOp<T> for GeluOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if is_infinite(x) {
return if x > <T as NumericElement>::ZERO {
x
} else {
-<T as NumericElement>::ZERO
};
}
let inv_sqrt_2 = T::from_f64(core::f64::consts::FRAC_1_SQRT_2);
let half = T::from_f64(0.5);
half * x * (-(x * inv_sqrt_2)).erfc()
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct GeluGradOp;
impl<T: RealScalar> UnaryOp<T> for GeluGradOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if is_infinite(x) {
return if x > <T as NumericElement>::ZERO {
<T as NumericElement>::ONE
} else {
<T as NumericElement>::ZERO
};
}
let inv_sqrt_2 = T::from_f64(core::f64::consts::FRAC_1_SQRT_2);
let inv_sqrt_2pi = T::from_f64(0.398_942_280_401_432_7);
let half = T::from_f64(0.5);
let neg_half = T::from_f64(-0.5);
half * (-(x * inv_sqrt_2)).erfc() + x * (neg_half * x * x).exp() * inv_sqrt_2pi
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct GeluTanhOp;
impl<T: RealScalar> UnaryOp<T> for GeluTanhOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if is_infinite(x) {
return if x > <T as NumericElement>::ZERO {
x
} else {
-<T as NumericElement>::ZERO
};
}
let z = gelu_tanh_arg(x);
x * stable_sigmoid(z + z)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct GeluTanhGradOp;
impl<T: RealScalar> UnaryOp<T> for GeluTanhGradOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if is_infinite(x) {
return if x > <T as NumericElement>::ZERO {
<T as NumericElement>::ONE
} else {
<T as NumericElement>::ZERO
};
}
let c0 = T::from_f64(0.797_884_560_802_865_4);
let c1 = T::from_f64(0.044715);
let z = gelu_tanh_arg(x);
let w = z + z;
let s = stable_sigmoid(w);
let one = <T as NumericElement>::ONE;
let two = one + one;
let three = two + one;
if w <= T::from_f64(GELU_TANH_GRAD_CROSSOVER) {
let one_minus_s = one - s;
if s * one_minus_s == <T as NumericElement>::ZERO {
return s;
}
s + two * x * s * one_minus_s * c0 * (one + three * c1 * x * x)
} else {
let one_minus_s = stable_sigmoid(-w);
let saturation = s * one_minus_s;
if saturation == <T as NumericElement>::ZERO {
return s;
}
s + two * x * saturation * c0 * (one + three * c1 * x * x)
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct SiluOp;
impl<T: RealScalar> UnaryOp<T> for SiluOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if is_infinite(x) {
return if x > <T as NumericElement>::ZERO {
x
} else {
-<T as NumericElement>::ZERO
};
}
x * stable_sigmoid(x)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct SiluGradOp;
impl<T: RealScalar> UnaryOp<T> for SiluGradOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if is_infinite(x) {
return if x > <T as NumericElement>::ZERO {
<T as NumericElement>::ONE
} else {
<T as NumericElement>::ZERO
};
}
let sig = stable_sigmoid(x);
let one_minus_sig = if x <= T::from_f64(SILU_GRAD_CROSSOVER) {
<T as NumericElement>::ONE - sig
} else {
stable_sigmoid(-x)
};
sig * (<T as NumericElement>::ONE + x * one_minus_sig)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct MishOp;
impl<T: RealScalar> UnaryOp<T> for MishOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if is_infinite(x) {
return if x > <T as NumericElement>::ZERO {
x
} else {
-<T as NumericElement>::ZERO
};
}
x * softplus_value(x).tanh()
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct MishGradOp;
impl<T: RealScalar> UnaryOp<T> for MishGradOp {
#[inline(always)]
fn apply(&self, x: T) -> T {
if is_infinite(x) {
return if x > <T as NumericElement>::ZERO {
<T as NumericElement>::ONE
} else {
<T as NumericElement>::ZERO
};
}
let sp = softplus_value(x);
let t = sp.tanh();
let one = <T as NumericElement>::ONE;
let one_minus_t_sq = if sp <= T::from_f64(MISH_GRAD_CROSSOVER) {
one - t * t
} else {
let two = one + one;
let four = two + two;
let u = (-two * sp).exp();
four * u / ((one + u) * (one + u))
};
t + x * one_minus_t_sq * stable_sigmoid(x)
}
}