concision_core/activate/utils/
funcs.rs

1/*
2    Appellation: utils <module>
3    Contrib: FL03 <jo3mccain@icloud.com>
4*/
5use ndarray::{Array, ArrayBase, Axis, Data, Dimension, RemoveAxis, ScalarOperand};
6use num_traits::{Float, One, Zero};
7
8/// the relu activation function:
9///
10/// ```math
11/// \mbox{f}(x) = \max(0, x)
12/// ```
13pub fn relu<T>(args: T) -> T
14where
15    T: PartialOrd + Zero,
16{
17    if args > T::zero() { args } else { T::zero() }
18}
19
20///
21///  ```math
22/// \frac{df}{dx}=\max(0,1)
23/// ```
24pub fn relu_derivative<T>(args: T) -> T
25where
26    T: PartialOrd + One + Zero,
27{
28    if args > T::zero() {
29        T::one()
30    } else {
31        T::zero()
32    }
33}
34/// the sigmoid activation function:
35///
36/// ```math
37/// f(x)=(1+e^{-x})^{-1}
38/// ```
39pub fn sigmoid<T>(args: T) -> T
40where
41    T: Float,
42{
43    (T::one() + args.neg().exp()).recip()
44}
45/// the derivative of the sigmoid function
46pub fn sigmoid_derivative<T>(args: T) -> T
47where
48    T: Float,
49{
50    let s = sigmoid(args);
51    s * (T::one() - s)
52}
53/// Softmax function:
54///
55/// ```math
56/// f(x_i) = \frac{e^{x_i}}{\sum_j e^{x_j}}
57/// ```
58pub fn softmax<A, S, D>(args: &ArrayBase<S, D, A>) -> Array<A, D>
59where
60    A: Float + ScalarOperand,
61    D: Dimension,
62    S: Data<Elem = A>,
63{
64    let e = args.exp();
65    &e / e.sum()
66}
67/// Softmax function along a specific axis:
68///
69/// ```math
70/// f(x_i) = \frac{e^{x_i}}{\sum_{j} e^{x_j}}
71/// ```
72pub fn softmax_axis<A, S, D>(args: &ArrayBase<S, D, A>, axis: usize) -> Array<A, D>
73where
74    A: Float + ScalarOperand,
75    D: RemoveAxis,
76    S: Data<Elem = A>,
77{
78    let axis = Axis(axis);
79    let e = args.exp();
80    &e / &e.sum_axis(axis)
81}
82/// Hyperbolic tangent
83///
84/// ```math
85/// f(x) = \frac{e^x - e^{-x}}{e^x + e^{-x}}
86/// ```
87pub fn tanh<T>(args: T) -> T
88where
89    T: Float,
90{
91    args.tanh()
92}
93/// the derivative of the tanh function
94pub fn tanh_derivative<T>(args: T) -> T
95where
96    T: Float,
97{
98    let t = tanh(args);
99    T::one() - t * t
100}
101
102/// the [`linear`] method is essentially a _passthrough_ method often used in simple models
103/// or layers where no activation is needed.
104pub const fn linear<T>(x: T) -> T {
105    x
106}
107
108/// the [`linear_derivative`] method always returns `1` as it is a simple, single variable
109/// function
110pub fn linear_derivative<T>() -> T
111where
112    T: One,
113{
114    <T>::one()
115}
116
117/// Heaviside activation function:
118///
119/// ```math
120/// H(x)=\begin{cases}1 &x\gt{0} \\ 0 &x\leq{0} \end{cases}
121/// ```
122pub fn heavyside<T>(x: T) -> T
123where
124    T: One + PartialOrd + Zero,
125{
126    if x > T::zero() { T::one() } else { T::zero() }
127}
concision_core/activate/utils/funcs.rs

concision_core/activate/utils/
funcs.rs
