rustyml 0.15.0

A high-performance machine learning & deep learning library in pure Rust, offering ML algorithms and neural network support
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

//! SELU activation layer that applies the scaled exponential linear unit elementwise and caches
//! the output for backpropagation

use crate::error::Error;
use crate::neural_network::Tensor;
use crate::neural_network::layers::TrainingParameters;
use crate::neural_network::layers::activation::{Activation, format_output_shape};
use crate::neural_network::layers::layer_weight::LayerWeight;
use crate::neural_network::layers::no_trainable_parameters_layer_functions;
use crate::neural_network::traits::Layer;

/// SELU (Scaled Exponential Linear Unit) activation layer
///
/// Applies `f(x) = scale * x` for `x > 0`, and `f(x) = scale * alpha * (e^x - 1)` at 0 and
/// below, keeping the original shape. The values `alpha = 1.6732632` and `scale = 1.0507010`
/// are fixed. Common inputs include 2D tensors for dense layers and 4D tensors for
/// convolutional layers
///
/// Klambauer et al. (2017) picked the 2 constants so that the activations keep a mean of 0
/// and a variance of 1 across layers. That self-normalizing property needs Lecun-normal
/// initialization and a wide enough layer
///
/// [`Activation::SELU`] provides the activation math. This layer only adds boundary
/// validation and the caching needed for backpropagation
///
/// # Examples
///
/// ```rust
/// use rustyml::neural_network::sequential::Sequential;
/// use rustyml::neural_network::layers::activation::selu::SELU;
/// use rustyml::neural_network::optimizers::*;
/// use rustyml::neural_network::losses::MeanSquaredError;
/// use ndarray::Array2;
///
/// // Create a 2D input tensor
/// let x = Array2::from_shape_vec((2, 3), vec![-1.0, 2.0, -3.0, 4.0, -5.0, 6.0])
///     .unwrap()
///     .into_dyn();
///
/// // Build a model with SELU activation
/// let mut model = Sequential::new();
/// model
///     .add(SELU::new())
///     .compile(SGD::new(0.01, 0.0, false, 0.0).unwrap(), MeanSquaredError::new());
///
/// // Forward propagation
/// let output = model.predict(&x);
///
/// // Output will be: [[-1.1113307, 2.101402, -1.6705688], [4.2028041, -1.7462534, 6.3042059]]
/// ```
#[derive(Debug)]
pub struct SELU {
    /// Cached activated output from the forward pass, used during backpropagation
    output_cache: Option<Tensor>,
}

impl SELU {
    /// Creates a new SELU activation layer
    ///
    /// # Returns
    ///
    /// - `Self` - A new `SELU` layer
    pub fn new() -> Self {
        SELU { output_cache: None }
    }
}

impl Default for SELU {
    fn default() -> Self {
        Self::new()
    }
}

impl Layer for SELU {
    fn forward(&mut self, input: &Tensor) -> Result<Tensor, Error> {
        if input.is_empty() {
            return Err(Error::empty_input("input tensor"));
        }

        let output = Activation::SELU.forward(input)?;

        // Cache activated output for backpropagation
        self.output_cache = Some(output.clone());

        Ok(output)
    }

    /// Inference forward (eval mode, writes no caches). See [`Layer::predict`]
    fn predict(&self, input: &Tensor) -> Result<Tensor, Error> {
        if input.is_empty() {
            return Err(Error::empty_input("input tensor"));
        }

        Activation::SELU.forward(input)
    }

    fn backward(&mut self, grad_output: &Tensor) -> Result<Tensor, Error> {
        if let Some(output) = &self.output_cache {
            // SELU preserves shape, so gradient must match the cached output
            if grad_output.shape() != output.shape() {
                return Err(Error::shape_mismatch(output.shape(), grad_output.shape()));
            }

            // SELU derivative is scale for x > 0, and scale * alpha * e^x for x <= 0
            Activation::SELU.backward(output, grad_output)
        } else {
            Err(Error::forward_pass_not_run("SELU"))
        }
    }

    fn layer_type(&self) -> &str {
        "SELU"
    }

    fn output_shape(&self) -> String {
        format_output_shape(&self.output_cache)
    }

    no_trainable_parameters_layer_functions!();
}