rustyml 0.16.0

A high-performance machine learning & deep learning library in pure Rust, offering ML algorithms and neural network support
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

//! Hard sigmoid activation layer
//!
//! The `HardSigmoid` struct holds only the shape recorded at build time, because the layer
//! takes no parameter. `UnaryLayer::forward` computes `Activation::HardSigmoid` and caches the
//! output during training. `UnaryLayer::backward` reads that cache to compute the gradient

use crate::error::Error;
use crate::neural_network::layers::ParamCounts;
use crate::neural_network::layers::activation::Activation;
use crate::neural_network::layers::validation::start_build;
use crate::neural_network::layers::{
    built_layer_shape_functions, no_trainable_parameters_layer_functions,
};
use crate::neural_network::traits::{LayerBase, UnaryLayer};
use crate::neural_network::{Ctx, Shape, Tensor};

/// Hard sigmoid activation layer
///
/// Applies `f(x) = clip(x/6 + 0.5, 0, 1)` elementwise to the input tensor, keeping the
/// original shape. Common inputs include 2D tensors for dense layers and 4D tensors for
/// convolutional layers
///
/// This function is a piecewise-linear approximation of the logistic sigmoid that needs no
/// exponential. The output is exactly 0 at `x <= -3` and exactly 1 at `x >= 3`, where the
/// gradient is 0
///
/// [`Activation::HardSigmoid`] provides the activation math. This layer only adds boundary
/// validation and the caching needed for backpropagation
///
/// # Examples
///
/// ```rust
/// use rustyml::neural_network::Shape;
/// use rustyml::neural_network::sequential::SequentialBuilder;
/// use rustyml::neural_network::layers::activation::hard_sigmoid::HardSigmoid;
/// use rustyml::neural_network::optimizers::*;
/// use rustyml::neural_network::losses::MeanSquaredError;
/// use ndarray::Array2;
///
/// // Create a 2D input tensor
/// let x = Array2::from_shape_vec((2, 3), vec![-1.0, 2.0, -3.0, 4.0, -5.0, 6.0])
///     .unwrap()
///     .into_dyn();
///
/// // Build a model with hard sigmoid activation
/// let mut model = SequentialBuilder::new()
///     .add(HardSigmoid::new())
///     .build(&Shape::known(x.shape()))
///     .unwrap();
/// model.compile(SGD::new(0.01, 0.0, false, 0.0).unwrap(), MeanSquaredError::new());
///
/// // Forward propagation
/// let output = model.predict(&x);
///
/// // Output will be: [[0.33333334, 0.83333337, 0.0], [1.0, 0.0, 1.0]]
/// ```
#[derive(Debug)]
pub struct HardSigmoid {
    /// Shape the layer was built for, batch axis first. `None` before the build
    built: Option<Shape>,
}

impl HardSigmoid {
    /// Creates a new hard sigmoid activation layer
    ///
    /// # Returns
    ///
    /// - `Self` - A new `HardSigmoid` layer
    pub fn new() -> Self {
        HardSigmoid { built: None }
    }
}

impl Default for HardSigmoid {
    fn default() -> Self {
        Self::new()
    }
}

impl LayerBase for HardSigmoid {
    fn layer_type(&self) -> &str {
        "HardSigmoid"
    }

    built_layer_shape_functions!();

    no_trainable_parameters_layer_functions!();
}

impl UnaryLayer for HardSigmoid {
    /// Records the shape the layer serves. The layer holds no array, so nothing is allocated
    fn build(&mut self, input: &Shape) -> Result<(), Error> {
        let Some(built) = start_build(&self.built, "HardSigmoid", input)? else {
            return Ok(());
        };
        self.compute_output_shape(&built)?;
        self.built = Some(built);
        Ok(())
    }

    fn forward(&self, input: &Tensor, ctx: &mut Ctx) -> Result<Tensor, Error> {
        if input.is_empty() {
            return Err(Error::empty_input("input tensor"));
        }

        let output = Activation::HardSigmoid.forward(input)?;

        if ctx.is_training() {
            ctx.push_cache("HardSigmoid", output.clone());
        }

        Ok(output)
    }

    fn backward(&self, grad_output: &Tensor, ctx: &mut Ctx) -> Result<Tensor, Error> {
        let output: Tensor = ctx.pop_cache("HardSigmoid")?;

        // Hard sigmoid preserves shape, so gradient must match the cached output
        if grad_output.shape() != output.shape() {
            return Err(Error::shape_mismatch(output.shape(), grad_output.shape()));
        }

        // Hard sigmoid derivative is 1/6 on the linear segment, and 0 on both saturated ends
        Activation::HardSigmoid.backward(&output, grad_output)
    }
}