Skip to main content

coeus_nn/activation/
basic.rs

1//! Basic activation functions and zero-parameter activation modules.
2//!
3//! This module exposes both functional helpers, such as [`relu`] and
4//! [`sigmoid`], and corresponding [`crate::module::Module`] wrappers for
5//! building models.
6
7use crate::module::{Module, ModuleError};
8use coeus_autograd::Var;
9use coeus_core::{Float, Scalar};
10
11/// Functional ReLU activation.
12#[inline]
13pub fn relu<T: Scalar, B: coeus_ops::BackendOps<T> + Default>(
14    input: &Var<T, B>,
15) -> Result<Var<T, B>, B::Error> {
16    coeus_autograd::relu(input)
17}
18
19/// ReLU activation module.
20#[derive(Clone, Debug, Default)]
21pub struct ReLU;
22
23/// Implements the [`crate::module::Module`] interface for [`ReLU`].
24impl<T: Scalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for ReLU {
25    #[inline]
26    fn parameters(&self) -> Vec<Var<T, B>> {
27        vec![]
28    }
29
30    #[inline]
31    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
32        Ok(relu(input)?)
33    }
34}
35
36/// Functional Sigmoid activation.
37#[inline]
38pub fn sigmoid<T: Float, B: coeus_ops::BackendOps<T> + Default>(
39    input: &Var<T, B>,
40) -> Result<Var<T, B>, B::Error> {
41    coeus_autograd::sigmoid(input)
42}
43
44/// Sigmoid activation module.
45#[derive(Clone, Debug, Default)]
46pub struct Sigmoid;
47
48/// Implements the [`crate::module::Module`] interface for [`Sigmoid`].
49impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Sigmoid {
50    #[inline]
51    fn parameters(&self) -> Vec<Var<T, B>> {
52        vec![]
53    }
54
55    #[inline]
56    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
57        Ok(sigmoid(input)?)
58    }
59}
60
61/// Functional Tanh activation.
62#[inline]
63pub fn tanh<T: Float, B: coeus_ops::BackendOps<T> + Default>(
64    input: &Var<T, B>,
65) -> Result<Var<T, B>, B::Error> {
66    coeus_autograd::tanh(input)
67}
68
69/// Tanh activation module.
70#[derive(Clone, Debug, Default)]
71pub struct Tanh;
72
73/// Implements the [`crate::module::Module`] interface for [`Tanh`].
74impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Tanh {
75    #[inline]
76    fn parameters(&self) -> Vec<Var<T, B>> {
77        vec![]
78    }
79
80    #[inline]
81    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
82        Ok(tanh(input)?)
83    }
84}
85
86/// Functional GELU activation.
87#[inline]
88pub fn gelu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
89    input: &Var<T, B>,
90) -> Result<Var<T, B>, B::Error> {
91    coeus_autograd::gelu(input)
92}
93
94/// GeLU activation module.
95#[derive(Clone, Debug, Default)]
96pub struct GeLU;
97
98/// Implements the [`crate::module::Module`] interface for [`GeLU`].
99impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for GeLU {
100    #[inline]
101    fn parameters(&self) -> Vec<Var<T, B>> {
102        vec![]
103    }
104
105    #[inline]
106    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
107        Ok(gelu(input)?)
108    }
109}
110
111/// Functional SiLU activation.
112#[inline]
113pub fn silu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
114    input: &Var<T, B>,
115) -> Result<Var<T, B>, B::Error> {
116    coeus_autograd::silu(input)
117}
118
119/// SiLU activation module.
120#[derive(Clone, Debug, Default)]
121pub struct SiLU;
122
123/// Implements the [`crate::module::Module`] interface for [`SiLU`].
124impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for SiLU {
125    #[inline]
126    fn parameters(&self) -> Vec<Var<T, B>> {
127        vec![]
128    }
129
130    #[inline]
131    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
132        Ok(silu(input)?)
133    }
134}
135
136/// Functional Mish activation.
137#[inline]
138pub fn mish<T: Float, B: coeus_ops::BackendOps<T> + Default>(
139    input: &Var<T, B>,
140) -> Result<Var<T, B>, B::Error> {
141    coeus_autograd::mish(input)
142}
143
144/// Mish activation module.
145#[derive(Clone, Debug, Default)]
146pub struct Mish;
147
148/// Implements the [`crate::module::Module`] interface for [`Mish`].
149impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Mish {
150    #[inline]
151    fn parameters(&self) -> Vec<Var<T, B>> {
152        vec![]
153    }
154
155    #[inline]
156    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
157        Ok(mish(input)?)
158    }
159}
160
161/// Public descriptor for the functional Hardsigmoid operation.
162#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
163pub struct HardsigmoidOp;
164
165impl HardsigmoidOp {
166    /// Operation name used by the autograd node.
167    pub const OP_NAME: &'static str = "hardsigmoid";
168}
169
170/// Functional Hardsigmoid activation.
171#[inline]
172pub fn hardsigmoid<T: Float, B: coeus_ops::BackendOps<T> + Default>(
173    input: &Var<T, B>,
174) -> Result<Var<T, B>, B::Error> {
175    coeus_autograd::hardsigmoid(input)
176}
177
178/// Hardsigmoid activation module.
179#[derive(Clone, Debug, Default)]
180pub struct Hardsigmoid;
181
182/// Implements the [`crate::module::Module`] interface for [`Hardsigmoid`].
183impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Hardsigmoid {
184    #[inline]
185    fn parameters(&self) -> Vec<Var<T, B>> {
186        vec![]
187    }
188
189    #[inline]
190    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
191        Ok(hardsigmoid(input)?)
192    }
193}
194
195/// Public descriptor for the functional Hardswish operation.
196#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
197pub struct HardswishOp;
198
199impl HardswishOp {
200    /// Operation name used by the autograd node.
201    pub const OP_NAME: &'static str = "hardswish";
202}
203
204/// Functional Hardswish activation.
205#[inline]
206pub fn hardswish<T: Float, B: coeus_ops::BackendOps<T> + Default>(
207    input: &Var<T, B>,
208) -> Result<Var<T, B>, B::Error> {
209    coeus_autograd::hardswish(input)
210}
211
212/// Hardswish activation module.
213#[derive(Clone, Debug, Default)]
214pub struct Hardswish;
215
216/// Implements the [`crate::module::Module`] interface for [`Hardswish`].
217impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Hardswish {
218    #[inline]
219    fn parameters(&self) -> Vec<Var<T, B>> {
220        vec![]
221    }
222
223    #[inline]
224    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
225        Ok(hardswish(input)?)
226    }
227}
228
229/// Public descriptor for the functional Softsign operation.
230#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
231pub struct SoftsignOp;
232
233impl SoftsignOp {
234    /// Operation name used by the autograd node.
235    pub const OP_NAME: &'static str = "softsign";
236}
237
238/// Functional Softsign activation.
239#[inline]
240pub fn softsign<T: Float, B: coeus_ops::BackendOps<T> + Default>(
241    input: &Var<T, B>,
242) -> Result<Var<T, B>, B::Error> {
243    coeus_autograd::softsign(input)
244}
245
246/// Softsign activation module.
247#[derive(Clone, Debug, Default)]
248pub struct Softsign;
249
250/// Implements the [`crate::module::Module`] interface for [`Softsign`].
251impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Softsign {
252    #[inline]
253    fn parameters(&self) -> Vec<Var<T, B>> {
254        vec![]
255    }
256
257    #[inline]
258    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
259        Ok(softsign(input)?)
260    }
261}
262
263/// Functional Softplus activation.
264#[inline]
265pub fn softplus<T: Float, B: coeus_ops::BackendOps<T> + Default>(
266    input: &Var<T, B>,
267) -> Result<Var<T, B>, B::Error> {
268    coeus_autograd::softplus(input)
269}
270
271/// Softplus activation module.
272#[derive(Clone, Debug, Default)]
273pub struct Softplus;
274
275/// Implements the [`crate::module::Module`] interface for [`Softplus`].
276impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Softplus {
277    #[inline]
278    fn parameters(&self) -> Vec<Var<T, B>> {
279        vec![]
280    }
281
282    #[inline]
283    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
284        Ok(softplus(input)?)
285    }
286}
287
288/// Functional LogSigmoid activation: `log(sigmoid(x)) = -softplus(-x)`.
289///
290/// Uses the numerically stable `-softplus(-x)` identity (avoids `log` of a tiny
291/// sigmoid); matches `torch.nn.functional.logsigmoid`.
292#[inline]
293pub fn log_sigmoid<T: Float, B: coeus_ops::BackendOps<T> + Default>(
294    input: &Var<T, B>,
295) -> Result<Var<T, B>, B::Error> {
296    coeus_autograd::neg(&coeus_autograd::softplus(&coeus_autograd::neg(input)?)?)
297}
298
299/// LogSigmoid activation module.
300#[derive(Clone, Debug, Default)]
301pub struct LogSigmoid;
302
303/// Implements the [`crate::module::Module`] interface for [`LogSigmoid`].
304impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for LogSigmoid {
305    #[inline]
306    fn parameters(&self) -> Vec<Var<T, B>> {
307        vec![]
308    }
309
310    #[inline]
311    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
312        Ok(log_sigmoid(input)?)
313    }
314}
315
316/// Functional Tanhshrink activation: `x - tanh(x)`.
317///
318/// Matches `torch.nn.functional.tanhshrink`.
319#[inline]
320pub fn tanhshrink<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
321    input: &Var<T, B>,
322) -> Result<Var<T, B>, B::Error> {
323    coeus_autograd::sub(input, &coeus_autograd::tanh(input)?)
324}
325
326/// Tanhshrink activation module.
327#[derive(Clone, Debug, Default)]
328pub struct Tanhshrink;
329
330/// Implements the [`crate::module::Module`] interface for [`Tanhshrink`].
331impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
332    for Tanhshrink
333{
334    #[inline]
335    fn parameters(&self) -> Vec<Var<T, B>> {
336        vec![]
337    }
338
339    #[inline]
340    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
341        Ok(tanhshrink(input)?)
342    }
343}