1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
//! Activations as layers, thin wrappers over the [`Tensor`] methods.
use crate::tensor::Tensor;
use super::module::Module;
macro_rules! activation {
($(#[$doc:meta])* $name:ident => $method:ident) => {
$(#[$doc])*
pub struct $name;
impl Module for $name {
fn forward(&self, input: &Tensor) -> Tensor {
input.$method()
}
}
};
}
activation!(/// `max(0, x)`.
ReLU => relu);
activation!(/// `1 / (1 + e^-x)`.
Sigmoid => sigmoid);
activation!(/// Hyperbolic tangent.
Tanh => tanh);
activation!(/// Exact GELU, the transformer default.
GELU => gelu);
activation!(/// SiLU / swish, `x ยท sigmoid(x)`.
SiLU => silu);
activation!(/// Softmax over the last dimension.
///
/// Skip it before [`cross_entropy`](super::cross_entropy), which has its own.
Softmax => softmax);
const DEFAULT_SLOPE: f32 = 0.01;
/// ReLU with a small slope for negative inputs.
pub struct LeakyReLU {
pub slope: f32,
}
impl LeakyReLU {
pub fn new(slope: f32) -> LeakyReLU {
LeakyReLU { slope }
}
}
impl Default for LeakyReLU {
fn default() -> Self {
LeakyReLU {
slope: DEFAULT_SLOPE,
}
}
}
impl Module for LeakyReLU {
fn forward(&self, input: &Tensor) -> Tensor {
input.leaky_relu(self.slope)
}
}