rustyml 0.17.0

Machine learning models, neural networks, metrics, and data utilities in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

//! Tanh (hyperbolic tangent) activation layer
//!
//! The `Tanh` struct holds only the shape recorded at build time, because the layer takes no
//! parameter. `UnaryLayer::forward` computes `Activation::Tanh` and caches the output during
//! training. `UnaryLayer::backward` reads that cache to compute the gradient

use crate::error::Error;
use crate::neural_network::layers::ParamCounts;
use crate::neural_network::layers::activation::{Activation, ActivationCache, ActivationInput};
use crate::neural_network::layers::validation::start_build;
use crate::neural_network::layers::{
    built_layer_shape_functions, no_trainable_parameters_layer_functions,
};
use crate::neural_network::traits::{LayerBase, UnaryLayer};
use crate::neural_network::{Ctx, Shape, Tensor};

/// Tanh (hyperbolic tangent) activation layer
///
/// Applies `tanh(x)` elementwise to the input tensor, mapping values to (-1, 1) and keeping the
/// original shape. Common inputs include 2D tensors for dense layers and 4D tensors for
/// convolutional layers
///
/// [`Activation::Tanh`] provides the activation math. This layer only adds boundary
/// validation and the caching needed for backpropagation
///
/// # Examples
///
/// ```rust
/// use rustyml::neural_network::Shape;
/// use rustyml::neural_network::sequential::SequentialBuilder;
/// use rustyml::neural_network::layers::activation::tanh::Tanh;
/// use rustyml::neural_network::optimizers::*;
/// use rustyml::neural_network::losses::*;
/// use ndarray::Array2;
///
/// // Create a 2D input tensor.
/// let x = Array2::from_shape_vec((2, 3), vec![-1.0, 2.0, -3.0, 4.0, -5.0, 6.0])
///     .unwrap()
///     .into_dyn();
///
/// // Build a model with Tanh activation.
/// let mut model = SequentialBuilder::new()
///     .add(Tanh::new())
///     .build(&Shape::known(x.shape()))
///     .unwrap();
/// model.compile(SGD::new(0.01, 0.0, false, 0.0).unwrap(), MeanSquaredError::new());
///
/// // Forward propagation.
/// let output = model.predict(&x);
///
/// // Output will be approximately: [[-0.76, 0.96, -0.99], [0.99, -0.99, 1.0]].
/// ```
#[derive(Debug)]
pub struct Tanh {
    /// Shape the layer was built for, batch axis first. `None` before the build
    built: Option<Shape>,
}

impl Tanh {
    /// Creates a new Tanh activation layer
    ///
    /// # Returns
    ///
    /// - `Self` - A new `Tanh` layer
    pub fn new() -> Self {
        Tanh { built: None }
    }
}

impl Default for Tanh {
    /// Creates the default Tanh layer.
    ///
    /// # Default Values
    ///
    /// Uses the configuration from [`Self::new`].
    ///
    /// # Returns
    ///
    /// - `Self` - The default layer.
    fn default() -> Self {
        Self::new()
    }
}

impl LayerBase for Tanh {
    fn layer_type(&self) -> &str {
        "Tanh"
    }

    built_layer_shape_functions!();

    no_trainable_parameters_layer_functions!();
}

impl UnaryLayer for Tanh {
    /// Records the shape the layer serves. The layer holds no array, so nothing is
    /// allocated
    fn build(&mut self, input: &Shape) -> Result<(), Error> {
        let Some(built) = start_build(&self.built, "Tanh", input)? else {
            return Ok(());
        };
        self.compute_output_shape(&built)?;
        self.built = Some(built);
        Ok(())
    }

    fn forward(&self, input: &Tensor, ctx: &mut Ctx) -> Result<Tensor, Error> {
        if input.is_empty() {
            return Err(Error::empty_input("input tensor"));
        }

        // Large-magnitude inputs saturate toward -1/+1 by construction
        let output = Activation::Tanh.forward(input)?;

        // The backward pass reads the output, so the cache holds a copy of it
        if ctx.is_training() {
            ctx.push_cache(
                "Tanh",
                ActivationCache::new(ActivationInput::Output, output.clone()),
            );
        }

        Ok(output)
    }

    fn backward(&self, grad_output: &Tensor, ctx: &mut Ctx) -> Result<Tensor, Error> {
        let cache: ActivationCache = ctx.pop_cache("Tanh")?;

        // Tanh derivative is 1 - tanh(x)^2
        Activation::Tanh.backward(&cache, grad_output)
    }
}