burn_core/nn/
leaky_relu.rs

1use crate as burn;
2use crate::config::Config;
3use crate::module::Module;
4use crate::module::{Content, DisplaySettings, ModuleDisplay};
5use crate::tensor::Tensor;
6use crate::tensor::backend::Backend;
7
8use crate::tensor::activation::leaky_relu;
9
10/// Leaky ReLu layer.
11///
12/// Should be created with [LeakyReluConfig](LeakyReluConfig).
13#[derive(Module, Clone, Debug)]
14#[module(custom_display)]
15pub struct LeakyRelu {
16    /// The negative slope.
17    pub negative_slope: f64,
18}
19/// Configuration to create a [Leaky Relu](LeakyRelu) layer using the [init function](LeakyReluConfig::init).
20#[derive(Config, Debug)]
21pub struct LeakyReluConfig {
22    /// The negative slope. Default is 0.01
23    #[config(default = "0.01")]
24    pub negative_slope: f64,
25}
26impl LeakyReluConfig {
27    /// Initialize a new [Leaky Relu](LeakyRelu) Layer
28    pub fn init(&self) -> LeakyRelu {
29        LeakyRelu {
30            negative_slope: self.negative_slope,
31        }
32    }
33}
34
35impl ModuleDisplay for LeakyRelu {
36    fn custom_settings(&self) -> Option<DisplaySettings> {
37        DisplaySettings::new()
38            .with_new_line_after_attribute(false)
39            .optional()
40    }
41
42    fn custom_content(&self, content: Content) -> Option<Content> {
43        content
44            .add("negative_slope", &self.negative_slope)
45            .optional()
46    }
47}
48
49impl LeakyRelu {
50    /// Forward pass for the Leaky ReLu layer.
51    ///
52    /// See [leaky_relu](crate::tensor::activation::leaky_relu) for more information.
53    ///
54    /// # Shapes
55    /// - input: `[..., any]`
56    /// - output: `[..., any]`
57    pub fn forward<B: Backend, const D: usize>(&self, input: Tensor<B, D>) -> Tensor<B, D> {
58        leaky_relu(input, self.negative_slope)
59    }
60}
61
62#[cfg(test)]
63mod tests {
64    use super::*;
65    use crate::TestBackend;
66    use crate::tensor::TensorData;
67    use burn_tensor::{Tolerance, ops::FloatElem};
68    type FT = FloatElem<TestBackend>;
69
70    #[test]
71    fn test_leaky_relu_forward() {
72        let device = <TestBackend as Backend>::Device::default();
73        let model: LeakyRelu = LeakyReluConfig::new().init();
74        let input =
75            Tensor::<TestBackend, 2>::from_data(TensorData::from([[0.4410, -0.2507]]), &device);
76        let out = model.forward(input);
77        let expected = TensorData::from([[0.4410, -0.002507]]);
78        out.to_data().assert_eq(&expected, false);
79    }
80    #[test]
81    fn test_leaky_relu_forward_multi_dim() {
82        let input = [
83            [
84                [-1.0222, 1.5810, 0.3457, -1.3530],
85                [0.0231, 0.8681, 0.2473, -0.0377],
86                [0.3520, -1.1199, 1.2219, 0.2804],
87            ],
88            [
89                [1.0002, 0.7259, 0.8779, 0.2084],
90                [1.5615, -0.1057, -0.4886, -1.5184],
91                [-0.5523, -0.2741, -0.0210, -1.1352],
92            ],
93        ];
94        let expected = TensorData::from([
95            [
96                [-1.0222e-02, 1.5810e+00, 3.457e-01, -1.3530e-02],
97                [2.31e-02, 8.681e-01, 2.473e-01, -3.77e-04],
98                [3.52e-01, -1.1199e-02, 1.2219e+00, 2.804e-01],
99            ],
100            [
101                [1.0002e+00, 7.259e-01, 8.779e-01, 2.084e-01],
102                [1.5615e+00, -1.057e-03, -4.886e-03, -1.5184e-02],
103                [-5.523e-03, -2.741e-03, -2.1e-04, -1.1352e-02],
104            ],
105        ]);
106
107        let device = <TestBackend as Backend>::Device::default();
108        let model: LeakyRelu = LeakyReluConfig::new().init();
109        let input_data = Tensor::<TestBackend, 3>::from_data(TensorData::from(input), &device);
110        let actual_output = model.forward(input_data);
111        actual_output
112            .to_data()
113            .assert_approx_eq::<FT>(&expected, Tolerance::default())
114    }
115
116    #[test]
117    fn display() {
118        let config = LeakyReluConfig::new().init();
119        assert_eq!(
120            alloc::format!("{config}"),
121            "LeakyRelu {negative_slope: 0.01}"
122        );
123    }
124}
burn_core/nn/leaky_relu.rs

burn_core/nn/
leaky_relu.rs
