Skip to main content

incin_core/nn/
activation.rs

1use crate::err::{Error, Result};
2use crate::nn::module::{Module, ShapeInfo, TrainMode};
3use crate::shapes::{DynShape, Shape};
4use crate::tensor::base::Tensor;
5use crate::tensor::device::Device;
6use crate::tensor::grad::RequiresGrad;
7use alloc::{
8    string::{String, ToString},
9    vec::Vec,
10};
11
12/// The Rectified Linear Unit (ReLU) activation function: `f(x) = max(0, x)`.
13///
14/// This is a stateless module with no learnable parameters.
15#[derive(Debug, Clone, Default)]
16pub struct ReLU;
17
18macro_rules! impl_stateless_shape_info {
19    ($($ty:ty),+ $(,)?) => {
20        $(impl ShapeInfo for $ty {
21            fn shape_info(&self) -> Option<String> {
22                None
23            }
24        })+
25    };
26}
27
28impl_stateless_shape_info!(ReLU);
29
30/// Stateless - no training-dependent behavior, opts in with the trait's
31/// default no-op so it can appear inside a `Sequential` alongside layers
32/// that do have one (e.g. `Dropout`).
33impl TrainMode for ReLU {}
34
35use crate::exec::catalog::op;
36use crate::tensor::backend::Execute;
37
38impl<
39    S: Shape + DynShape,
40    B: crate::tensor::backend::VariableBackend + Execute<op::Relu>,
41    G: RequiresGrad,
42> Module<Tensor<S, B, f32, G>> for ReLU
43where
44    <B as Execute<op::Relu>>::Output: Into<B::Storage<f32>>,
45{
46    /// The output tensor type produced by this module's forward pass.
47    type Output = Tensor<S, B, f32, G>;
48    /// The error type returned if the forward pass fails.
49    type Error = Error;
50
51    #[inline]
52    /// Runs the forward pass of this module on the given input.
53    fn forward(
54        &self,
55        x: Tensor<S, B, f32, G>,
56    ) -> core::result::Result<Tensor<S, B, f32, G>, Error> {
57        x.relu()
58    }
59}
60
61/// The Gaussian Error Linear Unit (GELU) activation function.
62///
63/// GELU is a smooth approximation to ReLU commonly used in transformer architectures.
64/// This is a stateless module with no learnable parameters.
65#[derive(Debug, Clone, Default)]
66pub struct GELU;
67
68impl_stateless_shape_info!(GELU);
69
70/// Stateless - no training-dependent behavior, opts in with the trait's
71/// default no-op so it can appear inside a `Sequential` alongside layers
72/// that do have one (e.g. `Dropout`).
73impl TrainMode for GELU {}
74
75impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Gelu>>
76    Module<Tensor<S, B>> for GELU
77where
78    <B as Execute<op::Gelu>>::Output: Into<B::Storage<f32>>,
79{
80    /// The output tensor type produced by this module's forward pass.
81    type Output = Tensor<S, B>;
82    /// The error type returned if the forward pass fails.
83    type Error = Error;
84
85    #[inline]
86    /// Runs the forward pass of this module on the given input.
87    fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
88        x.gelu()
89    }
90}
91
92/// The Swish (SiLU) activation function: `f(x) = x * sigmoid(x)`.
93///
94/// Swish is a smooth, non-monotonic function that consistently performs better than ReLU
95/// in deeper networks. This is a stateless module with no learnable parameters.
96#[derive(Debug, Clone, Default)]
97pub struct Swish;
98
99impl_stateless_shape_info!(Swish);
100
101/// Stateless - no training-dependent behavior, opts in with the trait's
102/// default no-op so it can appear inside a `Sequential` alongside layers
103/// that do have one (e.g. `Dropout`).
104impl TrainMode for Swish {}
105
106impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Swish>>
107    Module<Tensor<S, B>> for Swish
108where
109    <B as Execute<op::Swish>>::Output: Into<B::Storage<f32>>,
110{
111    /// The output tensor type produced by this module's forward pass.
112    type Output = Tensor<S, B>;
113    /// The error type returned if the forward pass fails.
114    type Error = Error;
115
116    #[inline]
117    /// Runs the forward pass of this module on the given input.
118    fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
119        x.swish()
120    }
121}
122
123/// The Mish activation function: `f(x) = x * tanh(softplus(x))`.
124///
125/// Mish is a smooth, continuous, non-monotonic function that can improve training dynamics.
126/// This is a stateless module with no learnable parameters.
127#[derive(Debug, Clone, Default)]
128pub struct Mish;
129
130impl_stateless_shape_info!(Mish);
131
132/// Stateless - no training-dependent behavior, opts in with the trait's
133/// default no-op so it can appear inside a `Sequential` alongside layers
134/// that do have one (e.g. `Dropout`).
135impl TrainMode for Mish {}
136
137impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Mish>>
138    Module<Tensor<S, B>> for Mish
139where
140    <B as Execute<op::Mish>>::Output: Into<B::Storage<f32>>,
141{
142    /// The output tensor type produced by this module's forward pass.
143    type Output = Tensor<S, B>;
144    /// The error type returned if the forward pass fails.
145    type Error = Error;
146
147    #[inline]
148    /// Runs the forward pass of this module on the given input.
149    fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
150        x.mish()
151    }
152}
153
154/// The Exponential Linear Unit (ELU) activation function.
155///
156/// ELU approaches a negative constant as the input gets smaller.
157/// This implementation hardcodes alpha to 1.0.
158/// This is a stateless module with no learnable parameters.
159#[derive(Debug, Clone, Default)]
160#[allow(clippy::upper_case_acronyms)]
161pub struct ELU;
162
163impl_stateless_shape_info!(ELU);
164
165/// Stateless - no training-dependent behavior, opts in with the trait's
166/// default no-op so it can appear inside a `Sequential` alongside layers
167/// that do have one (e.g. `Dropout`).
168impl TrainMode for ELU {}
169
170impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Elu>>
171    Module<Tensor<S, B>> for ELU
172where
173    <B as Execute<op::Elu>>::Output: Into<B::Storage<f32>>,
174{
175    /// The output tensor type produced by this module's forward pass.
176    type Output = Tensor<S, B>;
177    /// The error type returned if the forward pass fails.
178    type Error = Error;
179
180    #[inline]
181    /// Runs the forward pass of this module on the given input.
182    fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
183        x.elu()
184    }
185}
186
187/// The Softmax activation function, applied along a specified axis.
188///
189/// Converts a vector of raw logits into a probability distribution that sums to 1.
190///
191/// ## Parameters
192/// * `dim` - The axis along which the softmax normalization is applied.
193#[derive(Debug, Clone)]
194pub struct Softmax {
195    /// The axis along which softmax is applied.
196    pub dim: usize,
197}
198
199impl ShapeInfo for Softmax {
200    fn shape_info(&self) -> Option<String> {
201        None
202    }
203}
204
205impl Softmax {
206    /// Creates a new instance with default (statically inferred) shape arguments.
207    pub fn new(dim: usize) -> Self {
208        Self { dim }
209    }
210}
211
212/// Stateless - no training-dependent behavior, opts in with the trait's
213/// default no-op so it can appear inside a `Sequential` alongside layers
214/// that do have one (e.g. `Dropout`).
215impl TrainMode for Softmax {}
216
217impl<
218    S: Shape + DynShape + crate::shapes::RuntimeRankProjection,
219    B: crate::tensor::backend::VariableBackend
220        + crate::tensor::backend::Execute<crate::exec::catalog::op::Softmax>,
221> Module<Tensor<S, B>> for Softmax
222where
223    <B as crate::tensor::backend::Execute<crate::exec::catalog::op::Softmax>>::Output:
224        Into<B::Storage<f32>>,
225{
226    /// The output tensor type produced by this module's forward pass.
227    type Output = Tensor<S, B>;
228    /// The error type returned if the forward pass fails.
229    type Error = Error;
230
231    #[inline]
232    /// Runs the forward pass of this module on the given input.
233    fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
234        x.softmax(self.dim as isize)
235    }
236}
237
238/// The Sigmoid activation function: `f(x) = 1 / (1 + exp(-x))`.
239///
240/// Squashes each element into the range `(0, 1)`. This is a stateless module.
241#[derive(Debug, Clone, Default)]
242pub struct Sigmoid;
243
244impl_stateless_shape_info!(Sigmoid);
245
246/// Stateless - no training-dependent behavior, opts in with the trait's
247/// default no-op so it can appear inside a `Sequential` alongside layers
248/// that do have one (e.g. `Dropout`).
249impl TrainMode for Sigmoid {}
250
251impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Sigmoid>>
252    Module<Tensor<S, B>> for Sigmoid
253where
254    <B as Execute<op::Sigmoid>>::Output: Into<B::Storage<f32>>,
255{
256    /// The output tensor type produced by this module's forward pass.
257    type Output = Tensor<S, B>;
258    /// The error type returned if the forward pass fails.
259    type Error = Error;
260
261    #[inline]
262    /// Runs the forward pass of this module on the given input.
263    fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
264        x.sigmoid()
265    }
266}
267
268/// The Hyperbolic Tangent (Tanh) activation function: `f(x) = tanh(x)`.
269///
270/// Squashes each element into the range `(-1, 1)`. This is a stateless module.
271#[derive(Debug, Clone, Default)]
272pub struct Tanh;
273
274impl_stateless_shape_info!(Tanh);
275
276/// Stateless - no training-dependent behavior, opts in with the trait's
277/// default no-op so it can appear inside a `Sequential` alongside layers
278/// that do have one (e.g. `Dropout`).
279impl TrainMode for Tanh {}
280
281impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Tanh>>
282    Module<Tensor<S, B>> for Tanh
283where
284    <B as Execute<op::Tanh>>::Output: Into<B::Storage<f32>>,
285{
286    /// The output tensor type produced by this module's forward pass.
287    type Output = Tensor<S, B>;
288    /// The error type returned if the forward pass fails.
289    type Error = Error;
290
291    #[inline]
292    /// Runs the forward pass of this module on the given input.
293    fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
294        x.tanh()
295    }
296}
297
298macro_rules! impl_stateless_state_visitors {
299    ($($t:ty),+ $(,)?) => {$ (
300        impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitState<B> for $t {
301            fn visit_state<V: crate::nn::StateVisitor<B>>(
302                &self,
303                _: &crate::nn::StatePath,
304                _: &mut V,
305            ) -> crate::err::Result<()> { Ok(()) }
306        }
307
308        impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitStateMut<B> for $t {
309            fn visit_state_mut<V: crate::nn::StateMutVisitor<B>>(
310                &mut self,
311                _: &crate::nn::StatePath,
312                _: &mut V,
313            ) -> crate::err::Result<()> { Ok(()) }
314        }
315
316        impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitParameters<B> for $t {
317            fn visit_parameters<V: crate::nn::ParameterVisitor<B>>(
318                &self,
319                _: &crate::nn::StatePath,
320                _: &mut V,
321            ) -> crate::err::Result<()> { Ok(()) }
322        }
323    )+ };
324}
325
326impl_stateless_state_visitors!(ReLU, GELU, Swish, Mish, ELU, Softmax, Sigmoid, Tanh);
327
328impl crate::nn::module::NamedLayers for ReLU {
329    /// Returns the layer hierarchy rooted at this module for visualization.
330    fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
331        vec![crate::nn::module::LayerNode {
332            name: prefix.to_string(),
333            type_name: alloc::string::String::from("ReLU"),
334            shape_info: "".to_string(),
335            children: vec![],
336        }]
337    }
338}
339
340impl crate::nn::module::NamedLayers for GELU {
341    /// Returns the layer hierarchy rooted at this module for visualization.
342    fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
343        vec![crate::nn::module::LayerNode {
344            name: prefix.to_string(),
345            type_name: alloc::string::String::from("GELU"),
346            shape_info: "".to_string(),
347            children: vec![],
348        }]
349    }
350}
351
352impl crate::nn::module::NamedLayers for Swish {
353    /// Returns the layer hierarchy rooted at this module for visualization.
354    fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
355        vec![crate::nn::module::LayerNode {
356            name: prefix.to_string(),
357            type_name: alloc::string::String::from("Swish"),
358            shape_info: "".to_string(),
359            children: vec![],
360        }]
361    }
362}
363
364impl crate::nn::module::NamedLayers for Sigmoid {
365    /// Returns the layer hierarchy rooted at this module for visualization.
366    fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
367        vec![crate::nn::module::LayerNode {
368            name: prefix.to_string(),
369            type_name: alloc::string::String::from("Sigmoid"),
370            shape_info: "".to_string(),
371            children: vec![],
372        }]
373    }
374}
375
376impl crate::nn::module::NamedLayers for Tanh {
377    /// Returns the layer hierarchy rooted at this module for visualization.
378    fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
379        vec![crate::nn::module::LayerNode {
380            name: prefix.to_string(),
381            type_name: alloc::string::String::from("Tanh"),
382            shape_info: "".to_string(),
383            children: vec![],
384        }]
385    }
386}
387
388impl crate::nn::module::NamedLayers for Softmax {
389    /// Returns the layer hierarchy rooted at this module for visualization.
390    fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
391        vec![crate::nn::module::LayerNode {
392            name: prefix.to_string(),
393            type_name: alloc::string::String::from("Softmax"),
394            shape_info: format!("dim={}", self.dim),
395            children: vec![],
396        }]
397    }
398}
399
400macro_rules! impl_unit_to_device {
401    ($($t:ty),+) => {
402        $(
403            impl<B: crate::tensor::backend::VariableBackend, NewD: Device> crate::tensor::transfer::ToDevice<B, NewD> for $t {
404                type Output = $t;
405                fn to_device(self, _arg: &NewD::Arg) -> Result<Self::Output> {
406                    Ok(self)
407                }
408            }
409        )+
410    };
411}
412
413impl_unit_to_device!(ReLU, GELU, Swish, Mish, ELU, Softmax, Sigmoid, Tanh);