1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
//! Activation functions for neural network layers
use serde::{Deserialize, Serialize};
/// Activation function type
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum Activation {
/// Rectified Linear Unit: max(0, x)
ReLU,
/// Sigmoid: 1 / (1 + exp(-x))
Sigmoid,
/// Hyperbolic tangent
Tanh,
/// No activation (linear)
Linear,
}
impl Activation {
/// Apply activation function
#[must_use]
pub fn apply(&self, x: f32) -> f32 {
match self {
Self::ReLU => x.max(0.0),
Self::Sigmoid => {
#[cfg(feature = "ml")]
{
1.0 / (1.0 + libm::expf(-x))
}
#[cfg(not(feature = "ml"))]
{
1.0 / (1.0 + (-x).exp())
}
}
Self::Tanh => {
#[cfg(feature = "ml")]
{
libm::tanhf(x)
}
#[cfg(not(feature = "ml"))]
{
x.tanh()
}
}
Self::Linear => x,
}
}
/// Apply derivative for backpropagation
/// Takes the pre-activation value (z)
#[must_use]
pub fn derivative(&self, z: f32) -> f32 {
match self {
Self::ReLU => {
if z > 0.0 {
1.0
} else {
0.0
}
}
Self::Sigmoid => {
let s = self.apply(z);
s * (1.0 - s)
}
Self::Tanh => {
let t = self.apply(z);
1.0 - t * t
}
Self::Linear => 1.0,
}
}
/// Convert activation to byte representation for serialization
#[must_use]
pub const fn to_byte(self) -> u8 {
match self {
Self::ReLU => 0,
Self::Sigmoid => 1,
Self::Tanh => 2,
Self::Linear => 3,
}
}
/// Create activation from byte representation
#[must_use]
pub const fn from_byte(byte: u8) -> Self {
match byte {
0 => Self::ReLU,
1 => Self::Sigmoid,
2 => Self::Tanh,
_ => Self::Linear,
}
}
}